From 2722c86a3d621cd2855e9e461d209741243d2228 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 16 Aug 2022 13:34:45 -0700 Subject: [PATCH 01/25] replacing MultiModel Pipeline with SweepablePipeline --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 34 +-- .../API/BinaryClassificationExperiment.cs | 224 +++++++++++++++--- .../API/SweepableExtension.cs | 53 +++-- .../AutoMLExperiment/AutoMLExperiment.cs | 113 +++++++-- .../HyperParameterProposer.cs | 93 ++++++++ .../IHyperParameterProposer.cs | 14 ++ .../AutoMLExperiment/IDatasetManager.cs | 9 +- .../AutoMLExperiment/IMetricManager.cs | 36 ++- .../Runner/SweepablePipelineCVRunner.cs | 57 +++++ .../{ => Runner}/TrialRunner.cs | 108 --------- .../AutoMLExperiment/TrialResult.cs | 16 -- .../AutoMLExperiment/TrialRunnerFactory.cs | 12 +- .../AutoMLExperiment/TrialSettings.cs | 2 - .../HyperParameterProposer.cs | 47 ---- .../TrialSettingsProposer/PipelineProposer.cs | 92 ++----- .../AutoMLExperiment/TunerFactory.cs | 10 +- .../SweepableEstimator/ISweepable.cs | 4 +- .../SweepableEstimator/MultiModelPipeline.cs | 4 +- .../SweepableEstimator/SweepablePipeline.cs | 36 ++- ...sts.AutoFeaturizer_image_test.approved.txt | 1 + ...sts.AutoFeaturizer_image_test.received.txt | 51 ++++ ...ests.AutoFeaturizer_iris_test.approved.txt | 1 + ...ests.AutoFeaturizer_iris_test.received.txt | 35 +++ ...AutoFeaturizer_uci_adult_test.approved.txt | 1 + ...AutoFeaturizer_uci_adult_test.received.txt | 84 +++++++ ...Tests.ImagePathFeaturizerTest.approved.txt | 1 + ...Tests.ImagePathFeaturizerTest.received.txt | 42 ++++ ...stimatorAndBinaryClassifiers.approved.txt} | 1 + ...eFromIEstimatorAndRegressors.approved.txt} | 1 + ...orAndSweepableEstimatorArray.approved.txt} | 1 + ...EstimatorAndMultiClassifiers.approved.txt} | 1 + ...torAndSweepableEstimatorArray.approved.txt | 85 +++++++ ...rPipelineAndMultiClassifiers.approved.txt} | 1 + ...ineAndSweepableEstimatorArray.approved.txt | 89 +++++++ ...IEstimatorAndMultiClassifiers.approved.txt | 85 +++++++ .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 30 +++ .../SweepableExtensionTest.cs | 61 ++++- 37 files changed, 1175 insertions(+), 360 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs rename src/Microsoft.ML.AutoML/AutoMLExperiment/{ => Runner}/TrialRunner.cs (66%) delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt rename test/Microsoft.ML.AutoML.Tests/ApprovalTests/{SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt => SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt} (98%) rename test/Microsoft.ML.AutoML.Tests/ApprovalTests/{SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt => SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt} (98%) rename test/Microsoft.ML.AutoML.Tests/ApprovalTests/{SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt => SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt} (98%) rename test/Microsoft.ML.AutoML.Tests/ApprovalTests/{SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt => SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt} (98%) create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt rename test/Microsoft.ML.AutoML.Tests/ApprovalTests/{SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt => SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt} (98%) create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index 9e63a004bf..510a1761cf 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -326,7 +326,7 @@ public AutoMLExperiment CreateExperiment(AutoMLExperiment.AutoMLExperimentSettin /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. /// if provided, use it as search space for sdca, otherwise the default search space will be used. /// - public SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + public SweepablePipeline BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -377,7 +377,7 @@ public SweepableEstimator[] BinaryClassification(string labelColumnName = Defaul res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } - return res.ToArray(); + return new SweepablePipeline().Append(res.ToArray()); } /// @@ -402,7 +402,7 @@ public SweepableEstimator[] BinaryClassification(string labelColumnName = Defaul /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. /// if provided, use it as search space for sdca, otherwise the default search space will be used. /// - public SweepableEstimator[] MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + public SweepablePipeline MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -455,7 +455,7 @@ public SweepableEstimator[] MultiClassification(string labelColumnName = Default res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } - return res.ToArray(); + return new SweepablePipeline().Append(res.ToArray()); } /// @@ -480,7 +480,7 @@ public SweepableEstimator[] MultiClassification(string labelColumnName = Default /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. /// if provided, use it as search space for sdca, otherwise the default search space will be used. /// - public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + public SweepablePipeline Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -531,7 +531,7 @@ public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNam res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } - return res.ToArray(); + return new SweepablePipeline().Append(res.ToArray()); } /// @@ -539,7 +539,7 @@ public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNam /// /// output column name. /// input column name. - internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inputColumnName) + internal SweepablePipeline TextFeaturizer(string outputColumnName, string inputColumnName) { var option = new FeaturizeTextOption { @@ -547,7 +547,7 @@ internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inp OutputColumnName = outputColumnName, }; - return new[] { SweepableEstimatorFactory.CreateFeaturizeText(option) }; + return new SweepablePipeline().Append(new[] { SweepableEstimatorFactory.CreateFeaturizeText(option) }); } /// @@ -555,7 +555,7 @@ internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inp /// /// output column names. /// input column names. - internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, string[] inputColumnNames) + internal SweepablePipeline NumericFeaturizer(string[] outputColumnNames, string[] inputColumnNames) { Contracts.CheckValue(inputColumnNames, nameof(inputColumnNames)); Contracts.CheckValue(outputColumnNames, nameof(outputColumnNames)); @@ -566,7 +566,7 @@ internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, stri OutputColumnNames = outputColumnNames, }; - return new[] { SweepableEstimatorFactory.CreateReplaceMissingValues(replaceMissingValueOption) }; + return new SweepablePipeline().Append(new[] { SweepableEstimatorFactory.CreateReplaceMissingValues(replaceMissingValueOption) }); } /// @@ -574,7 +574,7 @@ internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, stri /// /// output column names. /// input column names. - internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, string[] inputColumnNames) + internal SweepablePipeline CatalogFeaturizer(string[] outputColumnNames, string[] inputColumnNames) { Contracts.Check(outputColumnNames.Count() == inputColumnNames.Count() && outputColumnNames.Count() > 0, "outputColumnNames and inputColumnNames must have the same length and greater than 0"); @@ -584,10 +584,10 @@ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, stri OutputColumnNames = outputColumnNames, }; - return new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) }; + return new SweepablePipeline().Append(new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) }); } - internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string inputColumnName) + internal SweepablePipeline ImagePathFeaturizer(string outputColumnName, string inputColumnName) { // load image => resize image (224, 224) => extract pixels => dnn featurizer var loadImageOption = new LoadImageOption @@ -617,7 +617,7 @@ internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string OutputColumnName = outputColumnName, }; - var pipeline = new MultiModelPipeline(); + var pipeline = new SweepablePipeline(); return pipeline.Append(SweepableEstimatorFactory.CreateLoadImages(loadImageOption)) .Append(SweepableEstimatorFactory.CreateResizeImages(resizeImageOption)) @@ -637,7 +637,7 @@ internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string /// columns that should be treated as image path. If not specified, it will automatically infer if a column is catalog or not. /// output feature column. /// columns that won't be included when featurizing, like label - public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) + public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) { Contracts.CheckValue(data, nameof(data)); @@ -705,7 +705,7 @@ public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = " /// column information. /// output feature column. /// A for featurization. - public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features") + public SweepablePipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features") { Contracts.CheckValue(data, nameof(data)); Contracts.CheckValue(columnInformation, nameof(columnInformation)); @@ -720,7 +720,7 @@ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInf var catalogFeatureColumnNames = catalogFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray(); var imagePathColumnNames = imagePathFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray(); - var pipeline = new MultiModelPipeline(); + var pipeline = new SweepablePipeline(); if (numericFeatureColumnNames.Length > 0) { pipeline = pipeline.Append(this.NumericFeaturizer(numericFeatureColumnNames, numericFeatureColumnNames)); diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 8e372528df..6ff2d71186 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -4,11 +4,14 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; +using static Microsoft.ML.TrainCatalogBase; namespace Microsoft.ML.AutoML { @@ -149,6 +152,7 @@ public sealed class BinaryClassificationExperiment : ExperimentBase Execute(IDataView _experiment.SetDataset(splitData.TrainSet, splitData.TestSet); } - MultiModelPipeline pipeline = new MultiModelPipeline(); if (preFeaturizer != null) { - pipeline = pipeline.Append(preFeaturizer); + _pipeline = preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) + .Append(Context.Auto().BinaryClassification(label, Features)); + } + else + { + _pipeline = Context.Auto().Featurizer(trainData, columnInformation, Features) + .Append(Context.Auto().BinaryClassification(label, Features)); } - pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) - .Append(Context.Auto().BinaryClassification(label, Features)); - _experiment.SetPipeline(pipeline); + _experiment.SetPipeline(_pipeline); var monitor = new BinaryClassificationTrialResultMonitor(); monitor.OnTrialCompleted += (o, e) => @@ -202,6 +209,8 @@ public override ExperimentResult Execute(IDataView progressHandler?.Report(detail); }; + _experiment.SetTrialRunnerFactory(); + _experiment.SetHyperParameterProposer(); _experiment.SetMonitor(monitor); _experiment.Run(); @@ -219,16 +228,18 @@ public override ExperimentResult Execute(IDataView _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); - MultiModelPipeline pipeline = new MultiModelPipeline(); if (preFeaturizer != null) { - pipeline = pipeline.Append(preFeaturizer); + _pipeline = preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) + .Append(Context.Auto().BinaryClassification(label, Features)); + } + else + { + _pipeline = Context.Auto().Featurizer(trainData, columnInformation, Features) + .Append(Context.Auto().BinaryClassification(label, Features)); } - pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__")) - .Append(Context.Auto().BinaryClassification(label, featureColumnName: Features)); - - _experiment.SetPipeline(pipeline); + _experiment.SetPipeline(_pipeline); var monitor = new BinaryClassificationTrialResultMonitor(); monitor.OnTrialCompleted += (o, e) => { @@ -237,6 +248,8 @@ public override ExperimentResult Execute(IDataView }; _experiment.SetMonitor(monitor); + _experiment.SetTrialRunnerFactory(); + _experiment.SetHyperParameterProposer(); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e)); @@ -274,16 +287,18 @@ public override CrossValidationExperimentResult Exe _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); - MultiModelPipeline pipeline = new MultiModelPipeline(); if (preFeaturizer != null) { - pipeline = pipeline.Append(preFeaturizer); + _pipeline = preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) + .Append(Context.Auto().BinaryClassification(label, Features)); + } + else + { + _pipeline = Context.Auto().Featurizer(trainData, columnInformation, Features) + .Append(Context.Auto().BinaryClassification(label, Features)); } - pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__")) - .Append(Context.Auto().BinaryClassification(label, featureColumnName: Features)); - - _experiment.SetPipeline(pipeline); + _experiment.SetPipeline(_pipeline); var monitor = new BinaryClassificationTrialResultMonitor(); monitor.OnTrialCompleted += (o, e) => @@ -294,6 +309,8 @@ public override CrossValidationExperimentResult Exe }; _experiment.SetMonitor(monitor); + _experiment.SetTrialRunnerFactory(); + _experiment.SetHyperParameterProposer(); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => ToCrossValidationRunDetail(e)); @@ -327,22 +344,16 @@ private protected override CrossValidationRunDetail private RunDetail ToRunDetail(BinaryClassificationTrialResult result) { - var pipeline = result.TrialSettings.Pipeline; - var trainerName = pipeline.ToString(); - var parameter = result.TrialSettings.Parameter; - var estimator = pipeline.BuildTrainingPipeline(Context, parameter); + var trainerName = result.TrialSettings.Pipeline.ToString(); var modelContainer = new ModelContainer(Context, result.Model); - return new RunDetail(trainerName, estimator, null, modelContainer, result.BinaryClassificationMetrics, result.Exception); + return new RunDetail(trainerName, result.Pipeline, null, modelContainer, result.BinaryClassificationMetrics, result.Exception); } private CrossValidationRunDetail ToCrossValidationRunDetail(BinaryClassificationTrialResult result) { - var pipeline = result.TrialSettings.Pipeline; - var trainerName = pipeline.ToString(); - var parameter = result.TrialSettings.Parameter; - var estimator = pipeline.BuildTrainingPipeline(Context, parameter); + var trainerName = result.TrialSettings.Pipeline.ToString(); var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult(new ModelContainer(Context, m.Model), m.Metrics, result.Exception)); - return new CrossValidationRunDetail(trainerName, estimator, null, crossValidationResult); + return new CrossValidationRunDetail(trainerName, result.Pipeline, null, crossValidationResult); } } @@ -399,4 +410,163 @@ public void ReportRunningTrial(TrialSettings setting) { } } + + internal class BinaryClassificationCVRunner : ITrialRunner + { + private readonly MLContext _context; + private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; + + private readonly EciHyperParameterProposer _proposer; + + public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer) + { + _context = context; + _datasetManager = datasetManager; + _metricManager = metricManager; + _proposer = proposer; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (_datasetManager is CrossValidateDatasetManager datasetSettings + && _metricManager is BinaryMetricManager metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); + var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new BinaryClassificationTrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + BinaryClassificationMetrics = res.Metrics, + CrossValidationMetrics = metrics, + Pipeline = pipeline, + }; + } + + throw new ArgumentException(); + } + } + + + internal class BinaryClassificationTrainTestRunner : ITrialRunner + { + private readonly MLContext _context; + private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; + private readonly EciHyperParameterProposer _proposer; + + public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer) + { + _context = context; + _metricManager = metricManager; + _datasetManager = datasetManager; + _proposer = proposer; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + if (_datasetManager is TrainTestDatasetManager datasetSettings + && _metricManager is BinaryMetricManager metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var metric = metricSettings.Metric switch + { + BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new BinaryClassificationTrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + BinaryClassificationMetrics = metrics, + Pipeline = pipeline, + }; + } + + throw new ArgumentException(); + } + } + + + internal class BinaryExperimentTrialRunnerFactory : ITrialRunnerFactory + { + private readonly IServiceProvider _provider; + + public BinaryExperimentTrialRunnerFactory(IServiceProvider provider) + { + _provider = provider; + } + + public ITrialRunner CreateTrialRunner() + { + var datasetManager = _provider.GetService(); + var metricManager = _provider.GetService(); + + ITrialRunner runner = (datasetManager, metricManager) switch + { + (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), + (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), + _ => throw new NotImplementedException(), + }; + + return runner; + } + } + + /// + /// TrialResult with Binary Classification Metrics + /// + internal class BinaryClassificationTrialResult : TrialResult + { + public BinaryClassificationMetrics BinaryClassificationMetrics { get; set; } + + public IEnumerable> CrossValidationMetrics { get; set; } + + public Exception Exception { get; set; } + + public bool IsSucceed { get => Exception == null; } + + public bool IsCrossValidation { get => CrossValidationMetrics == null; } + + public EstimatorChain Pipeline { get; set; } + } } diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index 006e35ee6e..0c483392d7 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -16,49 +16,66 @@ public static SweepableEstimatorPipeline Append(this SweepableEstimatorPipeline return pipeline.Append(new SweepableEstimator((context, parameter) => estimator1, new SearchSpace.SearchSpace())); } + public static SweepablePipeline Append(this SweepableEstimatorPipeline pipeline, SweepablePipeline sweepablePipeline) + { + var res = new SweepablePipeline(); + foreach (var estimator in pipeline.Estimators) + { + res = res.Append(estimator); + } + + return res.Append(sweepablePipeline); + } + public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, SweepableEstimator estimator1) { return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); } + public static SweepablePipeline Append(this SweepableEstimator estimator, SweepablePipeline estimator1) + { + return new SweepablePipeline().Append(estimator).Append(estimator1); + } + public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, IEstimator estimator1) { return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); } - public static MultiModelPipeline Append(this IEstimator estimator, params SweepableEstimator[] estimators) + + public static SweepablePipeline Append(this IEstimator estimator, SweepablePipeline pipeline) { var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); - var multiModelPipeline = new MultiModelPipeline().Append(sweepableEstimator).Append(estimators); + var res = new SweepablePipeline().Append(sweepableEstimator).Append(pipeline); - return multiModelPipeline; + return res; } - public static MultiModelPipeline Append(this MultiModelPipeline pipeline, IEstimator estimator) + public static SweepablePipeline Append(this SweepableEstimator estimator, params SweepableEstimator[] estimators) { - var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); - var multiModelPipeline = pipeline.Append(sweepableEstimator); + var pipeline = new SweepablePipeline(); + pipeline = pipeline.Append(estimator); - return multiModelPipeline; + return pipeline.Append(estimators); } - public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) + public static SweepablePipeline Append(this IEstimator estimator, params SweepableEstimator[] estimators) { - var multiModelPipeline = new MultiModelPipeline(); - foreach (var estimator in pipeline.Estimators) - { - multiModelPipeline = multiModelPipeline.Append(estimator); - } + var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); + var pipeline = new SweepablePipeline().Append(sweepableEstimator).Append(estimators); - return multiModelPipeline.Append(estimators); + return pipeline; } - public static MultiModelPipeline Append(this SweepableEstimator estimator, params SweepableEstimator[] estimators) + public static SweepablePipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) { - var multiModelPipeline = new MultiModelPipeline(); - multiModelPipeline = multiModelPipeline.Append(estimator); + var res = new SweepablePipeline(); + foreach (var estimator in pipeline.Estimators) + { + res = res.Append(estimator); + } - return multiModelPipeline.Append(estimators); + return res.Append(estimators); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index afc8dba450..58ba1e59f9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -14,6 +14,7 @@ namespace Microsoft.ML.AutoML { public class AutoMLExperiment { + internal const string PipelineSearchspaceName = "_pipeline_"; private readonly AutoMLExperimentSettings _settings; private readonly MLContext _context; private double _bestError = double.MaxValue; @@ -24,6 +25,11 @@ public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { _context = context; _settings = settings; + if (_settings.SearchSpace == null) + { + _settings.SearchSpace = new SearchSpace.SearchSpace(); + } + _serviceCollection = new ServiceCollection(); } @@ -32,16 +38,20 @@ private void InitializeServiceCollection() _serviceCollection.TryAddSingleton(_context); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddSingleton(); _serviceCollection.TryAddSingleton(); _serviceCollection.TryAddTransient(); + _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddScoped(); - _serviceCollection.TryAddScoped(); + _serviceCollection.TryAddScoped(); + } + + private void Initialize() + { + InitializeServiceCollection(); } public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) @@ -79,6 +89,7 @@ public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) }; _serviceCollection.AddSingleton(datasetManager); + _serviceCollection.AddSingleton(datasetManager); return this; } @@ -114,9 +125,13 @@ public AutoMLExperiment SetMonitor(IMonitor monitor) return this; } - public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) + internal AutoMLExperiment SetPipeline(SweepablePipeline pipeline) { - _settings.Pipeline = pipeline; + _settings.SearchSpace[PipelineSearchspaceName] = pipeline.SearchSpace; + _serviceCollection.AddSingleton(pipeline); + + SetTrialRunnerFactory(); + return this; } @@ -142,9 +157,59 @@ public AutoMLExperiment SetTrialRunner(ITrialRunner runner) return this; } + internal AutoMLExperiment SetTrialRunnerFactory() + where TTrialRunnerFactory : ITrialRunnerFactory + { + var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), typeof(TTrialRunnerFactory), ServiceLifetime.Singleton); + if (_serviceCollection.Contains(descriptor)) + { + _serviceCollection.Replace(descriptor); + } + else + { + _serviceCollection.Add(descriptor); + } + + return this; + } + + internal AutoMLExperiment SetHyperParameterProposer(THyperParameterProposer proposer) + where THyperParameterProposer : class, IHyperParameterProposer + { + _serviceCollection.TryAddSingleton(proposer); + var descriptor = new ServiceDescriptor(typeof(IHyperParameterProposer), proposer); + if (_serviceCollection.Contains(descriptor)) + { + _serviceCollection.Replace(descriptor); + } + else + { + _serviceCollection.Add(descriptor); + } + + return this; + } + + internal AutoMLExperiment SetHyperParameterProposer() + where THyperParameterProposer : class, IHyperParameterProposer + { + _serviceCollection.TryAddSingleton(); + var descriptor = new ServiceDescriptor(typeof(IHyperParameterProposer), typeof(THyperParameterProposer), ServiceLifetime.Singleton); + if (_serviceCollection.Contains(descriptor)) + { + _serviceCollection.Replace(descriptor); + } + else + { + _serviceCollection.Add(descriptor); + } + + return this; + } + public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) { - var res = new MultiModelPipeline(); + var res = new SweepablePipeline(); foreach (var e in pipeline.Estimators) { res = res.Append(e); @@ -157,13 +222,9 @@ public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { - var metricManager = new BinaryMetricManager() - { - Metric = metric, - PredictedColumn = predictedColumn, - LabelColumn = labelColumn, - }; + var metricManager = new BinaryMetricManager(metric, predictedColumn, labelColumn); _serviceCollection.AddSingleton(metricManager); + _serviceCollection.AddSingleton(metricManager); SetIsMaximizeMetric(metricManager.IsMaximize); return this; @@ -227,12 +288,11 @@ public async Task RunAsync(CancellationToken ct = default) _context.CancelExecution(); }); - InitializeServiceCollection(); + Initialize(); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); var trialNum = 0; - var pipelineProposer = serviceProvider.GetService(); - var hyperParameterProposer = serviceProvider.GetService(); + var hyperParameterProposer = serviceProvider.GetService(); var runnerFactory = serviceProvider.GetService(); while (true) @@ -247,7 +307,6 @@ public async Task RunAsync(CancellationToken ct = default) TrialId = trialNum++, }; - setting = pipelineProposer.Propose(setting); setting = hyperParameterProposer.Propose(setting); monitor.ReportRunningTrial(setting); var runner = runnerFactory.CreateTrialRunner(); @@ -257,7 +316,6 @@ public async Task RunAsync(CancellationToken ct = default) var trialResult = runner.Run(setting, serviceProvider); monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); - pipelineProposer.Update(setting, trialResult); var error = _settings.IsMaximizeMetric ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) @@ -267,21 +325,26 @@ public async Task RunAsync(CancellationToken ct = default) monitor.ReportBestTrial(trialResult); } } + catch (OperationCanceledException) + { + break; + } catch (Exception ex) { - if (cts.Token.IsCancellationRequested) - { - break; - } - else + monitor.ReportFailTrial(setting, ex); + + if (!cts.IsCancellationRequested && _bestTrialResult == null) { // TODO // it's questionable on whether to abort the entire training process // for a single fail trial. We should make it an option and only exit // when error is fatal (like schema mismatch). - monitor.ReportFailTrial(setting, ex); throw; } + else + { + break; + } } } @@ -303,11 +366,11 @@ private void ValidateSettings() public class AutoMLExperimentSettings : ExperimentSettings { - public MultiModelPipeline Pipeline { get; set; } - public int? Seed { get; set; } public bool IsMaximizeMetric { get; set; } + + internal SearchSpace.SearchSpace SearchSpace { get; set; } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs new file mode 100644 index 0000000000..f6424108f6 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.ML.AutoML +{ + /// + /// propose hyper parameter using ECI index. + /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer + /// is more likely to be picked. + /// + internal class EciHyperParameterProposer : IHyperParameterProposer + { + private readonly Dictionary _tuners; + private readonly IServiceProvider _provider; + private readonly PipelineProposer _pipelineProposer; + // this dictionary records the schema for each trial. + // the key is trial id, and value is the schema for that trial. + private readonly Dictionary _schemasLookupMap; + + public EciHyperParameterProposer(PipelineProposer pipelineProposer, IServiceProvider provider) + { + _tuners = new Dictionary(); + _pipelineProposer = pipelineProposer; + _provider = provider; + _schemasLookupMap = new Dictionary(); + } + + public TrialSettings Propose(TrialSettings settings) + { + (var pipeline, var schema) = _pipelineProposer.ProposePipeline(settings); + settings.Pipeline = pipeline; + var tunerFactory = _provider.GetService(); + if (!_tuners.ContainsKey(schema)) + { + var t = tunerFactory.CreateTuner(settings); + _tuners.Add(schema, t); + } + + var tuner = _tuners[schema]; + var parameter = tuner.Propose(settings); + settings.Parameter = parameter; + _schemasLookupMap[settings.TrialId] = schema; + return settings; + } + + public void Update(TrialSettings settings, TrialResult result) + { + if (_schemasLookupMap.TryGetValue(settings.TrialId, out var schema)) + { + if (_tuners.TryGetValue(schema, out var tuner)) + { + tuner.Update(result); + } + + _pipelineProposer.Update(settings, result, schema); + } + } + } + + internal class NestedSearchSpaceHyperParameterProposer : IHyperParameterProposer + { + private readonly ITuner _tuner; + private readonly SweepablePipeline _pipeline; + + public NestedSearchSpaceHyperParameterProposer(SweepablePipeline pipeline, ITunerFactory tunerFactory) + { + this._tuner = tunerFactory.CreateTuner(null); + this._pipeline = pipeline; + } + + public TrialSettings Propose(TrialSettings settings) + { + var parameter = _tuner.Propose(settings); + settings.Parameter = parameter; + var keys = parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType().Replace(" ", string.Empty).Split('*'); + var estimators = keys.Select(k => _pipeline.Estimators[k]); + settings.Pipeline = new SweepableEstimatorPipeline(estimators); + + return settings; + } + + public void Update(TrialSettings parameter, TrialResult result) + { + _tuner.Update(result); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs new file mode 100644 index 0000000000..a29d4d4d28 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML +{ + internal interface IHyperParameterProposer : ITrialSettingsProposer + { + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index ac057be17d..3ea14dd01b 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -12,6 +12,13 @@ public interface IDatasetManager { } + internal interface ICrossValidateDatasetManager + { + int? Fold { get; set; } + + IDataView Dataset { get; set; } + } + public class TrainTestDatasetManager : IDatasetManager { public IDataView TrainDataset { get; set; } @@ -19,7 +26,7 @@ public class TrainTestDatasetManager : IDatasetManager public IDataView TestDataset { get; set; } } - public class CrossValidateDatasetManager : IDatasetManager + public class CrossValidateDatasetManager : IDatasetManager, ICrossValidateDatasetManager { public IDataView Dataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index 84316b2ba5..df4cb17eb9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -14,8 +14,22 @@ internal interface IMetricManager bool IsMaximize { get; } } - internal class BinaryMetricManager : IMetricManager + internal interface IEvaluateMetricManager : IMetricManager { + double Evaluate(MLContext context, IDataView eval); + + string MetricName { get; } + } + + internal class BinaryMetricManager : IEvaluateMetricManager + { + public BinaryMetricManager(BinaryClassificationMetric metric, string predictedColumn, string labelColumn) + { + Metric = metric; + PredictedColumn = predictedColumn; + LabelColumn = labelColumn; + } + public BinaryClassificationMetric Metric { get; set; } public string PredictedColumn { get; set; } @@ -34,6 +48,26 @@ internal class BinaryMetricManager : IMetricManager BinaryClassificationMetric.F1Score => true, _ => throw new NotImplementedException(), }; + + public string MetricName => Metric.ToString(); + + public double Evaluate(MLContext context, IDataView eval) + { + var metric = context.BinaryClassification.EvaluateNonCalibrated(eval, labelColumnName: LabelColumn, predictedLabelColumnName: PredictedColumn); + + return Metric switch + { + BinaryClassificationMetric.Accuracy => metric.Accuracy, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metric.AreaUnderPrecisionRecallCurve, + BinaryClassificationMetric.AreaUnderRocCurve => metric.AreaUnderRocCurve, + BinaryClassificationMetric.PositivePrecision => metric.PositivePrecision, + BinaryClassificationMetric.NegativePrecision => metric.NegativePrecision, + BinaryClassificationMetric.NegativeRecall => metric.NegativeRecall, + BinaryClassificationMetric.PositiveRecall => metric.PositivePrecision, + BinaryClassificationMetric.F1Score => metric.F1Score, + _ => throw new NotImplementedException(), + }; + } } internal class MultiClassMetricManager : IMetricManager diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs new file mode 100644 index 0000000000..fe46331d51 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + internal class SweepablePipelineCVRunner : ITrialRunner + { + private readonly MLContext _mLContext; + private readonly IEvaluateMetricManager _metricManager; + private readonly ICrossValidateDatasetManager _crossValidateDatasetManager; + private readonly SweepablePipeline _pipeline; + + public SweepablePipelineCVRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, ICrossValidateDatasetManager crossValidateDatasetManager) + { + _mLContext = context; + _metricManager = metricManager; + _pipeline = pipeline; + _crossValidateDatasetManager = crossValidateDatasetManager; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); + var datasetSplit = _mLContext.Data.CrossValidationSplit(_crossValidateDatasetManager.Dataset, _crossValidateDatasetManager.Fold ?? 5); + var metrics = new List(); + var models = new List(); + foreach (var split in datasetSplit) + { + var model = mlnetPipeline.Fit(split.TrainSet); + var eval = model.Transform(split.TestSet); + metrics.Add(_metricManager.Evaluate(_mLContext, eval)); + models.Add(model); + } + + stopWatch.Stop(); + + return new TrialResult + { + Metric = metrics.Average(), + Model = models.First(), + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + TrialSettings = settings, + }; + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs similarity index 66% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs index d765599267..341e2ed9ac 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs @@ -15,114 +15,6 @@ public interface ITrialRunner TrialResult Run(TrialSettings settings, IServiceProvider provider = null); } - internal class BinaryClassificationCVRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _datasetManager = datasetManager; - _metricManager = metricManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is BinaryMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch - { - BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new BinaryClassificationTrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - BinaryClassificationMetrics = res.Metrics, - CrossValidationMetrics = metrics, - }; - } - - throw new ArgumentException(); - } - } - - internal class BinaryClassificationTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is BinaryMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var metric = metricSettings.Metric switch - { - BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new BinaryClassificationTrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - BinaryClassificationMetrics = metrics, - }; - } - - throw new ArgumentException(); - } - } internal class MultiClassificationTrainTestRunner : ITrialRunner { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index 0b8625f37e..b74e3c8578 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -20,20 +20,4 @@ public class TrialResult public double DurationInMilliseconds { get; set; } } - - /// - /// TrialResult with Binary Classification Metrics - /// - internal class BinaryClassificationTrialResult : TrialResult - { - public BinaryClassificationMetrics BinaryClassificationMetrics { get; set; } - - public IEnumerable> CrossValidationMetrics { get; set; } - - public Exception Exception { get; set; } - - public bool IsSucceed { get => Exception == null; } - - public bool IsCrossValidation { get => CrossValidationMetrics == null; } - } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index 8bba47d321..7734f0e4e1 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -31,11 +31,12 @@ public CustomRunnerFactory(ITrialRunner runner) } } - internal class TrialRunnerFactory : ITrialRunnerFactory + + internal class SweepablePipelineTrialRunnerFactory : ITrialRunnerFactory { private readonly IServiceProvider _provider; - public TrialRunnerFactory(IServiceProvider provider) + public SweepablePipelineTrialRunnerFactory(IServiceProvider provider) { _provider = provider; } @@ -47,12 +48,7 @@ public TrialRunnerFactory(IServiceProvider provider) ITrialRunner? runner = (datasetManager, metricManager) switch { - (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), - (CrossValidateDatasetManager, MultiClassMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, MultiClassMetricManager) => _provider.GetService(), - (CrossValidateDatasetManager, RegressionMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, RegressionMetricManager) => _provider.GetService(), + (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), _ => throw new NotImplementedException(), }; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs index 19294ffde9..7896ea5e9e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs @@ -12,8 +12,6 @@ public class TrialSettings public SweepableEstimatorPipeline Pipeline { get; set; } - public string Schema { get; set; } - public Parameter Parameter { get; set; } public AutoMLExperiment.AutoMLExperimentSettings ExperimentSettings { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs deleted file mode 100644 index ee091027b2..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.DependencyInjection; - -namespace Microsoft.ML.AutoML -{ - internal class HyperParameterProposer : ITrialSettingsProposer - { - private readonly Dictionary _tuners; - private readonly IServiceProvider _provider; - - public HyperParameterProposer(IServiceProvider provider) - { - _tuners = new Dictionary(); - _provider = provider; - } - - public TrialSettings Propose(TrialSettings settings) - { - var tunerFactory = _provider.GetService(); - if (!_tuners.ContainsKey(settings.Schema)) - { - var t = tunerFactory.CreateTuner(settings); - _tuners.Add(settings.Schema, t); - } - - var tuner = _tuners[settings.Schema]; - var parameter = tuner.Propose(settings); - settings.Parameter = parameter; - - return settings; - } - - public void Update(TrialSettings settings, TrialResult result) - { - var schema = settings.Schema; - if (_tuners.TryGetValue(schema, out var tuner)) - { - tuner.Update(result); - } - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index 30134c0da0..63229b405c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -15,7 +15,7 @@ namespace Microsoft.ML.AutoML /// /// propose sweepable estimator pipeline from a group of candidates using eci in flaml (https://arxiv.org/abs/1911.04706) /// - internal class PipelineProposer : ISavableProposer + internal class PipelineProposer { private readonly Dictionary _estimatorCost; private Dictionary _learnerInitialCost; @@ -37,9 +37,10 @@ internal class PipelineProposer : ISavableProposer private double _globalBestError; private readonly Random _rand; - private MultiModelPipeline _multiModelPipeline; + private readonly SweepablePipeline _sweepablePipeline; + private readonly string[] _pipelineSchemas; - public PipelineProposer(AutoMLExperimentSettings settings) + public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSettings settings) { // this cost is used to initialize eci when started, the smaller the number, the less cost this trainer will use at start, and more likely it will be // picked. @@ -67,26 +68,25 @@ public PipelineProposer(AutoMLExperimentSettings settings) { EstimatorType.ImageClassificationMulti, 1 }, { EstimatorType.MatrixFactorization, 1 }, }; - _rand = new Random(settings.Seed ?? 0); - _multiModelPipeline = null; + _sweepablePipeline = sweepablePipeline; + _rand = new Random(settings.Seed ?? 0); + _pipelineSchemas = _sweepablePipeline.Schema.ToTerms().Select(t => t.ToString()).ToArray(); } - public TrialSettings Propose(TrialSettings settings) + public (SweepableEstimatorPipeline, string) ProposePipeline(TrialSettings settings) { - _multiModelPipeline = settings.ExperimentSettings.Pipeline; - _learnerInitialCost = _multiModelPipeline.PipelineIds.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _multiModelPipeline)); - var pipelineIds = _multiModelPipeline.PipelineIds; - + _learnerInitialCost = _pipelineSchemas.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _sweepablePipeline)); + string schema; if (_eci == null) { // initialize eci with the estimated cost and always start from pipeline which has lowest cost. - _eci = pipelineIds.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _multiModelPipeline)); - settings.Schema = _eci.OrderBy(kv => kv.Value).First().Key; + _eci = _pipelineSchemas.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _sweepablePipeline)); + schema = _eci.OrderBy(kv => kv.Value).First().Key; } else { - var probabilities = pipelineIds.Select(id => _eci[id]).ToArray(); + var probabilities = _pipelineSchemas.Select(id => _eci[id]).ToArray(); probabilities = ArrayMath.Inverse(probabilities); probabilities = ArrayMath.Normalize(probabilities); @@ -96,70 +96,26 @@ public TrialSettings Propose(TrialSettings settings) // selected pipeline id index int i; - for (i = 0; i != pipelineIds.Length; ++i) + for (i = 0; i != _pipelineSchemas.Length; ++i) { - sum += ((double[])probabilities)[i]; + sum += probabilities[i]; if (sum > randdouble) { break; } } - settings.Schema = pipelineIds[i]; - } - - settings.Pipeline = _multiModelPipeline.BuildSweepableEstimatorPipeline(settings.Schema); - return settings; - } - - public void SaveStatusToFile(string fileName) - { - using (var writer = new FileStream(fileName, FileMode.Create)) - { - SaveStatusToStream(writer); + schema = _pipelineSchemas[i]; } - } - public void SaveStatusToStream(Stream stream) - { - var status = new Status() - { - K1 = _k1, - K2 = _k2, - E1 = _e1, - E2 = _e2, - Eci = _eci, - GlobalBestError = _globalBestError, - }; - - using (var fileWriter = new StreamWriter(stream)) - { - var json = JsonConvert.SerializeObject(status); - fileWriter.Write(json); - } - } - public void LoadStatusFromFile(string fileName) - { - if (File.Exists(fileName)) - { - var json = File.ReadAllText(fileName); - var status = JsonConvert.DeserializeObject(json); - _k1 = status.K1; - _k2 = status.K2; - _e1 = status.E1; - _e2 = status.E2; - _eci = status.Eci; - _globalBestError = status.GlobalBestError; - } + return (_sweepablePipeline.BuildSweepableEstimatorPipeline(schema), schema); } - public void Update(TrialSettings parameter, TrialResult result) + public void Update(TrialSettings parameter, TrialResult result, string schema) { - var schema = parameter.Schema; var error = CaculateError(result.Metric, parameter.ExperimentSettings.IsMaximizeMetric); var duration = result.DurationInMilliseconds / 1000; - var pipelineIds = _multiModelPipeline.PipelineIds; var isSuccess = duration != 0; // if k1 is null, it means this is the first completed trial. @@ -173,10 +129,10 @@ public void Update(TrialSettings parameter, TrialResult result) { if (_k1 == null) { - _k1 = pipelineIds.ToDictionary(id => id, id => duration * _learnerInitialCost[id] / _learnerInitialCost[schema]); + _k1 = _pipelineSchemas.ToDictionary(id => id, id => duration * _learnerInitialCost[id] / _learnerInitialCost[schema]); _k2 = _k1.ToDictionary(kv => kv.Key, kv => kv.Value); - _e1 = pipelineIds.ToDictionary(id => id, id => error); - _e2 = pipelineIds.ToDictionary(id => id, id => 1.05 * error); + _e1 = _pipelineSchemas.ToDictionary(id => id, id => error); + _e2 = _pipelineSchemas.ToDictionary(id => id, id => 1.05 * error); _globalBestError = error; } else if (error >= _e1[schema]) @@ -227,15 +183,15 @@ private double CaculateError(double loss, bool isMaximize) return isMaximize ? 1 - loss : loss; } - private double GetEstimatedCostForPipeline(string kv, MultiModelPipeline multiModelPipeline) + private double GetEstimatedCostForPipeline(string schema, SweepablePipeline pipeline) { - var entity = Entity.FromExpression(kv); + var entity = Entity.FromExpression(schema); var estimatorTypes = entity.ValueEntities().Where(v => v is StringEntity s && s.Value != "Nil") .Select(v => { var s = v as StringEntity; - var estimator = multiModelPipeline.Estimators[s.Value]; + var estimator = pipeline.Estimators[s.Value]; return estimator.EstimatorType; }); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index 6acc5f37e6..6f0f3a2a3c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -27,8 +27,8 @@ public CostFrugalTunerFactory(IServiceProvider provider) public ITuner CreateTuner(TrialSettings settings) { var experimentSetting = _provider.GetService(); - var searchSpace = settings.Pipeline.SearchSpace; - var initParameter = settings.Pipeline.Parameter; + var searchSpace = settings?.Pipeline?.SearchSpace ?? experimentSetting.SearchSpace; + var initParameter = searchSpace.SampleFromFeatureSpace(searchSpace.Default); var isMaximize = experimentSetting.IsMaximizeMetric; return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); @@ -46,7 +46,8 @@ public RandomTunerFactory(IServiceProvider provider) public ITuner CreateTuner(TrialSettings settings) { - var searchSpace = settings.Pipeline.SearchSpace; + var experimentSetting = _provider.GetService(); + var searchSpace = settings?.Pipeline?.SearchSpace ?? experimentSetting.SearchSpace; return new RandomSearchTuner(searchSpace); } @@ -63,7 +64,8 @@ public GridSearchTunerFactory(IServiceProvider provider) public ITuner CreateTuner(TrialSettings settings) { - var searchSpace = settings.Pipeline.SearchSpace; + var experimentSetting = _provider.GetService(); + var searchSpace = settings?.Pipeline?.SearchSpace ?? experimentSetting.SearchSpace; return new GridSearchTuner(searchSpace); } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs index 5e7a6f681a..01a0a2935c 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs @@ -9,12 +9,12 @@ namespace Microsoft.ML.AutoML { - internal interface ISweepable + public interface ISweepable { public SearchSpace.SearchSpace SearchSpace { get; } } - internal interface ISweepable : ISweepable + public interface ISweepable : ISweepable where T : IEstimator { public T BuildFromOption(MLContext context, Parameter parameter); diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs index eb1b58d1a1..885a726b79 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs @@ -10,14 +10,14 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(MultiModelPipelineConverter))] - public class MultiModelPipeline + internal class MultiModelPipeline { private static readonly StringEntity _nilStringEntity = new StringEntity("Nil"); private static readonly EstimatorEntity _nilSweepableEntity = new EstimatorEntity(null); private readonly Dictionary _estimators; private readonly Entity _schema; - public MultiModelPipeline() + internal MultiModelPipeline() { _estimators = new Dictionary(); _schema = null; diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs index 37c4ac385e..91c099904f 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(SweepablePipelineConverter))] - internal class SweepablePipeline : ISweepable> + public class SweepablePipeline : ISweepable> { private readonly Entity _schema; private const string SchemaOption = "_SCHEMA_"; @@ -70,6 +70,11 @@ internal SweepablePipeline() _schema = null; } + internal SweepablePipeline(MultiModelPipeline pipeline) + : this(pipeline.Estimators, pipeline.Schema) + { + } + internal SweepablePipeline(Dictionary estimators, Entity schema, string currentSchema = null) { _estimators = estimators; @@ -85,13 +90,34 @@ internal SweepablePipeline(Dictionary estimators, En public EstimatorChain BuildFromOption(MLContext context, Parameter parameter) { _currentSchema = parameter[SchemaOption].AsType(); - var estimators = Entity.FromExpression(_currentSchema) + var pipeline = new EstimatorChain(); + var estimatorParameterPair = Entity.FromExpression(_currentSchema) .ValueEntities() .Where(e => e is StringEntity se && se.Value != "Nil") - .Select((se) => _estimators[((StringEntity)se).Value]); + .Select((se) => + { + var key = ((StringEntity)se).Value; + var estimator = _estimators[key]; + var param = parameter[key.Substring(1)]; + return (estimator, param); + }); + + foreach (var kv in estimatorParameterPair) + { + pipeline = pipeline.Append(kv.estimator.BuildFromOption(context, kv.param)); + } + + return pipeline; + } + + public SweepableEstimatorPipeline BuildSweepableEstimatorPipeline(string schema) + { + var pipelineNodes = Entity.FromExpression(schema) + .ValueEntities() + .Where(e => e is StringEntity se && se.Value != "Nil") + .Select((se) => _estimators[((StringEntity)se).Value]); - var pipeline = new SweepableEstimatorPipeline(estimators); - return pipeline.BuildTrainingPipeline(context, parameter); + return new SweepableEstimatorPipeline(pipelineNodes); } public SweepablePipeline Append(params ISweepable>[] sweepables) diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt index 4f137b8a8a..d2678afcc9 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * e2 * e3 * e4", + "currentSchema": "e0 * e1 * e2 * e3 * e4", "estimators": { "e0": { "estimatorType": "LoadImages", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt new file mode 100644 index 0000000000..d9697352c9 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt @@ -0,0 +1,51 @@ +{ + "schema": "e0 * e1 * e2 * e3 * e4", + "currentSchema": "e0 * e1 * e2 * e3 * e4", + "estimators": { + "e0": { + "estimatorType": "LoadImages", + "parameter": { + "OutputColumnName": "ImagePath", + "InputColumnName": "ImagePath" + } + }, + "e1": { + "estimatorType": "ResizeImages", + "parameter": { + "OutputColumnName": "ImagePath", + "InputColumnName": "ImagePath", + "ImageHeight": 224, + "ImageWidth": 224, + "CropAnchor": "Center", + "Resizing": "Fill" + } + }, + "e2": { + "estimatorType": "ExtractPixels", + "parameter": { + "OutputColumnName": "ImagePath", + "InputColumnName": "ImagePath", + "ColorsToExtract": "Rgb", + "OrderOfExtraction": "ARGB", + "OutputAsFloatArray": true + } + }, + "e3": { + "estimatorType": "DnnFeaturizerImage", + "parameter": { + "OutputColumnName": "ImagePath", + "InputColumnName": "ImagePath", + "ModelFactory": "resnet_18" + } + }, + "e4": { + "estimatorType": "Concatenate", + "parameter": { + "InputColumnNames": [ + "ImagePath" + ], + "OutputColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt index e6ec8ed089..d82286552b 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "ReplaceMissingValues", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt new file mode 100644 index 0000000000..0ec6f552d5 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt @@ -0,0 +1,35 @@ +{ + "schema": "e0 * e1", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "ReplaceMissingValues", + "parameter": { + "OutputColumnNames": [ + "col1", + "col2", + "col3", + "col4" + ], + "InputColumnNames": [ + "col1", + "col2", + "col3", + "col4" + ] + } + }, + "e1": { + "estimatorType": "Concatenate", + "parameter": { + "InputColumnNames": [ + "col1", + "col2", + "col3", + "col4" + ], + "OutputColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt index 63a64233a4..a265687a48 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 \u002B e2) * e3", + "currentSchema": "e0 * e1 * e3", "estimators": { "e0": { "estimatorType": "ReplaceMissingValues", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt new file mode 100644 index 0000000000..17c65fe6ad --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt @@ -0,0 +1,84 @@ +{ + "schema": "e0 * (e1 \u002B e2) * e3", + "currentSchema": "e0 * e1 * e3", + "estimators": { + "e0": { + "estimatorType": "ReplaceMissingValues", + "parameter": { + "OutputColumnNames": [ + "Features" + ], + "InputColumnNames": [ + "Features" + ] + } + }, + "e1": { + "estimatorType": "OneHotEncoding", + "parameter": { + "OutputColumnNames": [ + "Workclass", + "education", + "marital-status", + "occupation", + "relationship", + "ethnicity", + "sex", + "native-country-region" + ], + "InputColumnNames": [ + "Workclass", + "education", + "marital-status", + "occupation", + "relationship", + "ethnicity", + "sex", + "native-country-region" + ] + } + }, + "e2": { + "estimatorType": "OneHotHashEncoding", + "parameter": { + "OutputColumnNames": [ + "Workclass", + "education", + "marital-status", + "occupation", + "relationship", + "ethnicity", + "sex", + "native-country-region" + ], + "InputColumnNames": [ + "Workclass", + "education", + "marital-status", + "occupation", + "relationship", + "ethnicity", + "sex", + "native-country-region" + ] + } + }, + "e3": { + "estimatorType": "Concatenate", + "parameter": { + "InputColumnNames": [ + "Features", + "Workclass", + "education", + "marital-status", + "occupation", + "relationship", + "ethnicity", + "sex", + "native-country-region" + ], + "OutputColumnName": "OutputFeature" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt index 0b930b01aa..0a39f1f6fc 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * e2 * e3", + "currentSchema": "e0 * e1 * e2 * e3", "estimators": { "e0": { "estimatorType": "LoadImages", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt new file mode 100644 index 0000000000..bf06bdba19 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt @@ -0,0 +1,42 @@ +{ + "schema": "e0 * e1 * e2 * e3", + "currentSchema": "e0 * e1 * e2 * e3", + "estimators": { + "e0": { + "estimatorType": "LoadImages", + "parameter": { + "OutputColumnName": "imagePath", + "InputColumnName": "imagePath" + } + }, + "e1": { + "estimatorType": "ResizeImages", + "parameter": { + "OutputColumnName": "imagePath", + "InputColumnName": "imagePath", + "ImageHeight": 224, + "ImageWidth": 224, + "CropAnchor": "Center", + "Resizing": "Fill" + } + }, + "e2": { + "estimatorType": "ExtractPixels", + "parameter": { + "OutputColumnName": "imagePath", + "InputColumnName": "imagePath", + "ColorsToExtract": "Rgb", + "OrderOfExtraction": "ARGB", + "OutputAsFloatArray": true + } + }, + "e3": { + "estimatorType": "DnnFeaturizerImage", + "parameter": { + "OutputColumnName": "imagePath", + "InputColumnName": "imagePath", + "ModelFactory": "resnet_18" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt index d3be549eed..58efe42258 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt index 0e14cdb320..b02699087d 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt index 0e14cdb320..b02699087d 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt index 9661d5897c..258dc76181 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "FastForestBinary", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..3f6be2bbb3 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,85 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt index 05949bc69e..5ea3e5977e 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..5ea3e5977e --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,89 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..b02699087d --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,85 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 6860391591..c0a195ab6c 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -7,6 +7,7 @@ using System.Globalization; using System.Linq; using System.Threading; +using FluentAssertions; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; @@ -106,6 +107,35 @@ public void AutoFit_UCI_Adult_CrossValidation_10_Test() Assert.NotNull(result.BestRun.TrainerName); } + [LightGBMFact] + public void SweepablePipeline_AutoFit_UCI_Adult_CrossValidation_10_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var dataPath = DatasetUtil.GetUciAdultDataset(); + var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); + var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); + var trainData = textLoader.Load(dataPath); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(trainData, columnInference.ColumnInformation) + .Append(context.Auto().BinaryClassification(labelColumnName: DatasetUtil.UciAdultLabel)); + + experiment.SetPipeline(pipeline) + .SetDataset(trainData, 5) + .SetEvaluateMetric(BinaryClassificationMetric.Accuracy, DatasetUtil.UciAdultLabel) + .SetTrainingTimeInSeconds(10); + + var res = experiment.Run(); + res.Metric.Should().BeGreaterThan(0.5); + + } + [Theory] [InlineData(true)] [InlineData(false)] diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index db6c5fe8d3..e139061b36 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -49,6 +50,7 @@ public void CreateSweepableEstimatorPipelineFromIEstimatorTest() var estimator = context.Transforms.Concatenate("output", "input"); var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + pipeline.Should().BeOfType(); pipeline.ToString().Should().Be("Unknown=>FastForestBinary"); } @@ -60,6 +62,7 @@ public void AppendIEstimatorToSweepabeEstimatorPipelineTest() var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); + pipeline.Should().BeOfType(); pipeline.ToString().Should().Be("Unknown=>FastForestBinary=>Unknown"); } @@ -86,10 +89,10 @@ public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorT [Fact] [UseReporter(typeof(DiffReporter))] [UseApprovalSubdirectory("ApprovalTests")] - public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() + public void CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(context.Auto().BinaryClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); @@ -99,10 +102,10 @@ public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() + public void CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(context.Auto().MultiClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); @@ -112,10 +115,10 @@ public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromIEstimatorAndRegressors() + public void CreateSweepablePipelineFromIEstimatorAndRegressors() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(context.Auto().MultiClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); @@ -125,7 +128,7 @@ public void CreateMultiModelPipelineFromIEstimatorAndRegressors() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() + public void CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers() { var context = new MLContext(); var pipeline = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()) @@ -138,15 +141,55 @@ public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers() + public void CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption())) .Append(context.Auto().MultiClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); Approvals.Verify(json); } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray() + { + var context = new MLContext(); + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") + .Append(SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption())) + .Append(context.Auto().MultiClassification().Estimators.Select(kv => kv.Value).ToArray()); + + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray() + { + var context = new MLContext(); + SweepablePipeline pipeline = SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption()) + .Append(context.Auto().MultiClassification().Estimators.Select(kv => kv.Value).ToArray()); + + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray() + { + var context = new MLContext(); + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().MultiClassification().Estimators.Select(kv => kv.Value).ToArray()); + + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } } } From b580667cce8461b5eb5304867ea6f45339898965 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 16 Aug 2022 16:38:36 -0700 Subject: [PATCH 02/25] remove pipeline in TrialSettings --- .../AutoMLMonitorKernelExtension.cs | 4 +- .../NotebookMonitor.cs | 6 ++- .../API/BinaryClassificationExperiment.cs | 21 +++++----- .../AutoMLExperiment/AutoMLExperiment.cs | 35 +++++++++-------- .../HyperParameterProposer.cs | 30 ++++++--------- .../AutoMLExperiment/IMonitor.cs | 13 ++++--- .../TrialSettingsProposer/PipelineProposer.cs | 24 +++++++++--- .../AutoMLExperiment/TunerFactory.cs | 38 ++++++++----------- .../SweepableEstimator/SweepablePipeline.cs | 38 ++++++++++++++----- 9 files changed, 117 insertions(+), 92 deletions(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index 7868a24b4c..8d06caad80 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -51,7 +51,7 @@ private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) var bestTrialParam = JsonSerializer.Serialize(monitor.BestTrial.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = true, }); summary.Add(h3("Best Trial")); summary.Add(p($"Id: {monitor.BestTrial.TrialSettings.TrialId}")); - summary.Add(p($"Trainer: {monitor.BestTrial.TrialSettings.Pipeline}".Replace("Unknown=>", ""))); + summary.Add(p($"Trainer: {monitor.SweepablePipeline.ToString(monitor.BestTrial.TrialSettings.Parameter)}".Replace("Unknown=>", ""))); summary.Add(p($"Parameters: {bestTrialParam}")); } if (monitor.ActiveTrial != null) @@ -61,7 +61,7 @@ private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) summary.Add(h3("Active Trial")); summary.Add(p($"Id: {monitor.ActiveTrial.TrialId}")); - summary.Add(p($"Trainer: {monitor.ActiveTrial.Pipeline}".Replace("Unknown=>", ""))); + summary.Add(p($"Trainer: {monitor.SweepablePipeline.ToString(monitor.ActiveTrial.Parameter)}".Replace("Unknown=>", ""))); summary.Add(p($"Parameters: {activeTrialParam}")); } diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index 6aea99367d..cd555ec14d 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -17,17 +17,19 @@ public class NotebookMonitor : IMonitor private readonly ActionThrottler _updateThrottler; private DisplayedValue _valueToUpdate; + public SweepablePipeline SweepablePipeline { get; private set;} public TrialResult BestTrial { get; set; } public TrialResult MostRecentTrial { get; set; } public TrialSettings ActiveTrial { get; set; } public List CompletedTrials { get; set; } public DataFrame TrialData { get; set; } - public NotebookMonitor() + public NotebookMonitor(SweepablePipeline pipeline) { CompletedTrials = new List(); TrialData = new DataFrame(new PrimitiveDataFrameColumn("Trial"), new PrimitiveDataFrameColumn("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters")); _updateThrottler = new ActionThrottler(Update, TimeSpan.FromSeconds(5)); + SweepablePipeline = pipeline; } public void ReportBestTrial(TrialResult result) @@ -48,7 +50,7 @@ public void ReportCompletedTrial(TrialResult result) { new KeyValuePair("Trial",result.TrialSettings.TrialId), new KeyValuePair("Metric", result.Metric), - new KeyValuePair("Trainer",result.TrialSettings.Pipeline.ToString().Replace("Unknown=>","")), + new KeyValuePair("Trainer",SweepablePipeline.ToString(result.TrialSettings.Parameter).Replace("Unknown=>","")), new KeyValuePair("Parameters",activeRunParam), }, true); diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 6ff2d71186..5fd296cada 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -344,14 +344,14 @@ private protected override CrossValidationRunDetail private RunDetail ToRunDetail(BinaryClassificationTrialResult result) { - var trainerName = result.TrialSettings.Pipeline.ToString(); + var trainerName = _pipeline.ToString(result.TrialSettings.Parameter); var modelContainer = new ModelContainer(Context, result.Model); return new RunDetail(trainerName, result.Pipeline, null, modelContainer, result.BinaryClassificationMetrics, result.Exception); } private CrossValidationRunDetail ToCrossValidationRunDetail(BinaryClassificationTrialResult result) { - var trainerName = result.TrialSettings.Pipeline.ToString(); + var trainerName = _pipeline.ToString(result.TrialSettings.Parameter); var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult(new ModelContainer(Context, m.Model), m.Metrics, result.Exception)); return new CrossValidationRunDetail(trainerName, result.Pipeline, null, crossValidationResult); } @@ -416,15 +416,14 @@ internal class BinaryClassificationCVRunner : ITrialRunner private readonly MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; + private readonly SweepablePipeline _pipeline; - private readonly EciHyperParameterProposer _proposer; - - public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer) + public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer, SweepablePipeline pipeline) { _context = context; _datasetManager = datasetManager; _metricManager = metricManager; - _proposer = proposer; + _pipeline = pipeline; } public TrialResult Run(TrialSettings settings, IServiceProvider provider) @@ -436,7 +435,7 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) var stopWatch = new Stopwatch(); stopWatch.Start(); var fold = datasetSettings.Fold ?? 5; - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); + var pipeline = _pipeline.BuildFromOption(_context, settings.Parameter); var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. @@ -476,14 +475,14 @@ internal class BinaryClassificationTrainTestRunner : ITrialRunner private readonly MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; - private readonly EciHyperParameterProposer _proposer; + private readonly SweepablePipeline _pipeline; - public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer) + public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer, SweepablePipeline pipeline) { _context = context; _metricManager = metricManager; _datasetManager = datasetManager; - _proposer = proposer; + _pipeline = pipeline; } public TrialResult Run(TrialSettings settings, IServiceProvider provider) @@ -493,7 +492,7 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) { var stopWatch = new Stopwatch(); stopWatch.Start(); - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); + var pipeline = _pipeline.BuildFromOption(_context, settings.Parameter); var model = pipeline.Fit(datasetSettings.TrainDataset); var eval = model.Transform(datasetSettings.TestDataset); var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 58ba1e59f9..109aa8bd29 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -37,7 +37,6 @@ private void InitializeServiceCollection() { _serviceCollection.TryAddSingleton(_context); _serviceCollection.TryAddSingleton(_settings); - _serviceCollection.TryAddSingleton(); _serviceCollection.TryAddSingleton(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); @@ -110,7 +109,8 @@ public AutoMLExperiment SetTunerFactory() return this; } - public AutoMLExperiment SetMonitor(IMonitor monitor) + public AutoMLExperiment SetMonitor(TMonitor monitor) + where TMonitor : class, IMonitor { var descriptor = new ServiceDescriptor(typeof(IMonitor), monitor); if (_serviceCollection.Contains(descriptor)) @@ -125,19 +125,28 @@ public AutoMLExperiment SetMonitor(IMonitor monitor) return this; } - internal AutoMLExperiment SetPipeline(SweepablePipeline pipeline) + public AutoMLExperiment SetMonitor() { - _settings.SearchSpace[PipelineSearchspaceName] = pipeline.SearchSpace; - _serviceCollection.AddSingleton(pipeline); - - SetTrialRunnerFactory(); + var descriptor = new ServiceDescriptor(typeof(IMonitor), typeof(TMonitor), ServiceLifetime.Singleton); + if (_serviceCollection.Contains(descriptor)) + { + _serviceCollection.Replace(descriptor); + } + else + { + _serviceCollection.Add(descriptor); + } return this; } - public AutoMLExperiment SetIsMaximizeMetric(bool isMaximize) + internal AutoMLExperiment SetPipeline(SweepablePipeline pipeline) { - _settings.IsMaximizeMetric = isMaximize; + _settings.SearchSpace[PipelineSearchspaceName] = pipeline.SearchSpace; + _serviceCollection.AddSingleton(pipeline); + + SetTrialRunnerFactory(); + SetMonitor(); return this; } @@ -225,7 +234,6 @@ public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, str var metricManager = new BinaryMetricManager(metric, predictedColumn, labelColumn); _serviceCollection.AddSingleton(metricManager); _serviceCollection.AddSingleton(metricManager); - SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -239,7 +247,6 @@ public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); - SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -253,7 +260,6 @@ public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelC LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); - SetIsMaximizeMetric(metricManager.IsMaximize); return this; } @@ -294,6 +300,7 @@ public async Task RunAsync(CancellationToken ct = default) var trialNum = 0; var hyperParameterProposer = serviceProvider.GetService(); var runnerFactory = serviceProvider.GetService(); + var metricManager = serviceProvider.GetService(); while (true) { @@ -317,7 +324,7 @@ public async Task RunAsync(CancellationToken ct = default) monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); - var error = _settings.IsMaximizeMetric ? 1 - trialResult.Metric : trialResult.Metric; + var error = metricManager.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) { _bestTrialResult = trialResult; @@ -368,8 +375,6 @@ public class AutoMLExperimentSettings : ExperimentSettings { public int? Seed { get; set; } - public bool IsMaximizeMetric { get; set; } - internal SearchSpace.SearchSpace SearchSpace { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs index f6424108f6..19eb3c22b9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs @@ -17,49 +17,42 @@ namespace Microsoft.ML.AutoML internal class EciHyperParameterProposer : IHyperParameterProposer { private readonly Dictionary _tuners; - private readonly IServiceProvider _provider; private readonly PipelineProposer _pipelineProposer; // this dictionary records the schema for each trial. // the key is trial id, and value is the schema for that trial. - private readonly Dictionary _schemasLookupMap; + private readonly IMetricManager _metricManager; - public EciHyperParameterProposer(PipelineProposer pipelineProposer, IServiceProvider provider) + public EciHyperParameterProposer(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) { _tuners = new Dictionary(); - _pipelineProposer = pipelineProposer; - _provider = provider; - _schemasLookupMap = new Dictionary(); + _pipelineProposer = new PipelineProposer(sweepablePipeline, settings, metricManager); + _metricManager = metricManager; } public TrialSettings Propose(TrialSettings settings) { - (var pipeline, var schema) = _pipelineProposer.ProposePipeline(settings); - settings.Pipeline = pipeline; - var tunerFactory = _provider.GetService(); + (var searchSpace, var schema) = _pipelineProposer.ProposeSearchSpace(); if (!_tuners.ContainsKey(schema)) { - var t = tunerFactory.CreateTuner(settings); + var t = new CostFrugalTuner(searchSpace, searchSpace.SampleFromFeatureSpace(searchSpace.Default), !_metricManager.IsMaximize); _tuners.Add(schema, t); } var tuner = _tuners[schema]; var parameter = tuner.Propose(settings); settings.Parameter = parameter; - _schemasLookupMap[settings.TrialId] = schema; return settings; } public void Update(TrialSettings settings, TrialResult result) { - if (_schemasLookupMap.TryGetValue(settings.TrialId, out var schema)) + var schema = settings.Parameter["_SCHEMA_"].AsType(); + if (_tuners.TryGetValue(schema, out var tuner)) { - if (_tuners.TryGetValue(schema, out var tuner)) - { - tuner.Update(result); - } - - _pipelineProposer.Update(settings, result, schema); + tuner.Update(result); } + + _pipelineProposer.Update(settings, result, schema); } } @@ -80,7 +73,6 @@ public TrialSettings Propose(TrialSettings settings) settings.Parameter = parameter; var keys = parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType().Replace(" ", string.Empty).Split('*'); var estimators = keys.Select(k => _pipeline.Estimators[k]); - settings.Pipeline = new SweepableEstimatorPipeline(estimators); return settings; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index cc9cbe0035..5ffdd396fb 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -29,34 +29,35 @@ internal class MLContextMonitor : IMonitor private readonly IServiceProvider _serviceProvider; private readonly IChannel _logger; private readonly List _completedTrials; - - public MLContextMonitor(MLContext context, IServiceProvider provider) + private readonly SweepablePipeline _pipeline; + public MLContextMonitor(MLContext context, SweepablePipeline pipeline, IServiceProvider provider) { _context = context; _serviceProvider = provider; _logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); _completedTrials = new List(); + _pipeline = pipeline; } public void ReportBestTrial(TrialResult result) { - _logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + _logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)}"); } public void ReportCompletedTrial(TrialResult result) { - _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline} - Duration: {result.DurationInMilliseconds}"); + _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds}"); _completedTrials.Add(result); } public void ReportFailTrial(TrialSettings settings, Exception exception = null) { - _logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {settings.Pipeline}"); + _logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {_pipeline.ToString(settings.Parameter)}"); } public void ReportRunningTrial(TrialSettings setting) { - _logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {setting.Pipeline}"); + _logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {_pipeline.ToString(setting.Parameter)}"); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index 63229b405c..79ff2dcf78 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -7,7 +7,9 @@ using System.IO; using System.Linq; using Microsoft.ML.AutoML.CodeGen; +using Microsoft.ML.SearchSpace.Option; using Newtonsoft.Json; +using Tensorflow; using static Microsoft.ML.AutoML.AutoMLExperiment; namespace Microsoft.ML.AutoML @@ -38,9 +40,10 @@ internal class PipelineProposer private readonly Random _rand; private readonly SweepablePipeline _sweepablePipeline; + private readonly IMetricManager _metricManager; private readonly string[] _pipelineSchemas; - public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSettings settings) + public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSettings settings, IMetricManager metricManager) { // this cost is used to initialize eci when started, the smaller the number, the less cost this trainer will use at start, and more likely it will be // picked. @@ -70,11 +73,21 @@ public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSet }; _sweepablePipeline = sweepablePipeline; - _rand = new Random(settings.Seed ?? 0); + + if (settings.Seed.HasValue) + { + _rand = new Random(settings.Seed.Value); + } + else + { + _rand = new Random(); + } + _pipelineSchemas = _sweepablePipeline.Schema.ToTerms().Select(t => t.ToString()).ToArray(); + _metricManager = metricManager; } - public (SweepableEstimatorPipeline, string) ProposePipeline(TrialSettings settings) + public (SearchSpace.SearchSpace, string) ProposeSearchSpace() { _learnerInitialCost = _pipelineSchemas.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _sweepablePipeline)); string schema; @@ -108,13 +121,12 @@ public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSet schema = _pipelineSchemas[i]; } - - return (_sweepablePipeline.BuildSweepableEstimatorPipeline(schema), schema); + return (_sweepablePipeline.BuildSweepableEstimatorPipeline(schema).SearchSpace, schema); } public void Update(TrialSettings parameter, TrialResult result, string schema) { - var error = CaculateError(result.Metric, parameter.ExperimentSettings.IsMaximizeMetric); + var error = CaculateError(result.Metric, _metricManager.IsMaximize); var duration = result.DurationInMilliseconds / 1000; var isSuccess = duration != 0; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index 6f0f3a2a3c..3412143eea 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -17,57 +17,51 @@ public interface ITunerFactory internal class CostFrugalTunerFactory : ITunerFactory { - private readonly IServiceProvider _provider; + private readonly SearchSpace.SearchSpace _searchSpace; + private readonly IMetricManager _metricManager; - public CostFrugalTunerFactory(IServiceProvider provider) + public CostFrugalTunerFactory(AutoMLExperiment.AutoMLExperimentSettings settings, IMetricManager metricManager) { - _provider = provider; + _searchSpace = settings.SearchSpace; + _metricManager = metricManager; } public ITuner CreateTuner(TrialSettings settings) { - var experimentSetting = _provider.GetService(); - var searchSpace = settings?.Pipeline?.SearchSpace ?? experimentSetting.SearchSpace; - var initParameter = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var isMaximize = experimentSetting.IsMaximizeMetric; + var initParameter = _searchSpace.SampleFromFeatureSpace(_searchSpace.Default); + var isMaximize = _metricManager.IsMaximize; - return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); + return new CostFrugalTuner(_searchSpace, initParameter, !isMaximize); } } internal class RandomTunerFactory : ITunerFactory { - private readonly IServiceProvider _provider; + private readonly SearchSpace.SearchSpace _searchSpace; - public RandomTunerFactory(IServiceProvider provider) + public RandomTunerFactory(AutoMLExperiment.AutoMLExperimentSettings settings) { - _provider = provider; + _searchSpace = settings.SearchSpace; } public ITuner CreateTuner(TrialSettings settings) { - var experimentSetting = _provider.GetService(); - var searchSpace = settings?.Pipeline?.SearchSpace ?? experimentSetting.SearchSpace; - - return new RandomSearchTuner(searchSpace); + return new RandomSearchTuner(_searchSpace); } } internal class GridSearchTunerFactory : ITunerFactory { - private readonly IServiceProvider _provider; + private readonly SearchSpace.SearchSpace _searchSpace; - public GridSearchTunerFactory(IServiceProvider provider) + public GridSearchTunerFactory(AutoMLExperiment.AutoMLExperimentSettings settings) { - _provider = provider; + _searchSpace = settings.SearchSpace; } public ITuner CreateTuner(TrialSettings settings) { - var experimentSetting = _provider.GetService(); - var searchSpace = settings?.Pipeline?.SearchSpace ?? experimentSetting.SearchSpace; - - return new GridSearchTuner(searchSpace); + return new GridSearchTuner(_searchSpace); } } } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs index 91c099904f..d70c48868c 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs @@ -28,12 +28,11 @@ public SearchSpace.SearchSpace SearchSpace get { var searchSpace = new SearchSpace.SearchSpace(); - var kvPairs = _estimators.Select((e, i) => new KeyValuePair(i.ToString(), e.Value.SearchSpace)); - foreach (var kv in kvPairs) + foreach (var kv in _estimators) { if (kv.Value != null) { - searchSpace.Add(kv.Key, kv.Value); + searchSpace.Add(kv.Key, kv.Value.SearchSpace); } } @@ -98,7 +97,7 @@ public EstimatorChain BuildFromOption(MLContext context, Parameter { var key = ((StringEntity)se).Value; var estimator = _estimators[key]; - var param = parameter[key.Substring(1)]; + var param = parameter[key]; return (estimator, param); }); @@ -110,14 +109,14 @@ public EstimatorChain BuildFromOption(MLContext context, Parameter return pipeline; } - public SweepableEstimatorPipeline BuildSweepableEstimatorPipeline(string schema) + public SweepablePipeline BuildSweepableEstimatorPipeline(string schema) { - var pipelineNodes = Entity.FromExpression(schema) - .ValueEntities() + var entity = Entity.FromExpression(schema); + var pipelineNodes = entity.ValueEntities() .Where(e => e is StringEntity se && se.Value != "Nil") - .Select((se) => _estimators[((StringEntity)se).Value]); + .ToDictionary((se) => se.ToString(), (se) => _estimators[((StringEntity)se).Value]); - return new SweepableEstimatorPipeline(pipelineNodes); + return new SweepablePipeline(pipelineNodes, entity, schema); } public SweepablePipeline Append(params ISweepable>[] sweepables) @@ -154,6 +153,27 @@ public SweepablePipeline Append(params ISweepable>[] sw return AppendEntity(false, entity); } + public string ToString(Parameter parameter) + { + if (parameter.TryGetValue(AutoMLExperiment.PipelineSearchspaceName, out var pipelineParameter)) + { + var schema = pipelineParameter["_SCHEMA_"].AsType(); + var estimatorStrings = Entity.FromExpression(_currentSchema) + .ValueEntities() + .Where(e => e is StringEntity se && se.Value != "Nil") + .Select((se) => + { + var key = ((StringEntity)se).Value; + var estimator = _estimators[key]; + return estimator.EstimatorType.ToString(); + }); + + return string.Join("=>", estimatorStrings); + } + + return string.Empty; + } + private SweepablePipeline AppendEntity(bool allowSkip, Entity entity) { var estimators = _estimators.ToDictionary(x => x.Key, x => x.Value); From 192b266082a9c8d7700c2fd8503b23221fb45d77 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 16 Aug 2022 17:17:32 -0700 Subject: [PATCH 03/25] clean up --- .../API/BinaryClassificationExperiment.cs | 10 ++-- .../AutoMLExperiment/AutoMLExperiment.cs | 4 +- ...Proposer.cs => EciCfoParameterProposer.cs} | 43 ++++------------ .../IHyperParameterProposer.cs | 5 +- ...NestedSearchSpaceHyperParameterProposer.cs | 35 +++++++++++++ .../PipelineProposer.cs | 0 .../AutoMLExperiment/IDatasetManager.cs | 9 +++- .../SweepablePipelineTrainTestRunner.cs | 49 ++++++++++++++++++ .../AutoMLExperiment/TrialRunnerFactory.cs | 1 + .../ITrialSettingsProposer.cs | 20 -------- .../AutoMLExperimentTests.cs | 50 +++++++++++++++++++ 11 files changed, 166 insertions(+), 60 deletions(-) rename src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/{HyperParameterProposer.cs => EciCfoParameterProposer.cs} (58%) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs rename src/Microsoft.ML.AutoML/AutoMLExperiment/{TrialSettingsProposer => HyperParameterProposers}/PipelineProposer.cs (100%) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 5fd296cada..fccecedf98 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -210,7 +210,7 @@ public override ExperimentResult Execute(IDataView }; _experiment.SetTrialRunnerFactory(); - _experiment.SetHyperParameterProposer(); + _experiment.SetHyperParameterProposer(); _experiment.SetMonitor(monitor); _experiment.Run(); @@ -249,7 +249,7 @@ public override ExperimentResult Execute(IDataView _experiment.SetMonitor(monitor); _experiment.SetTrialRunnerFactory(); - _experiment.SetHyperParameterProposer(); + _experiment.SetHyperParameterProposer(); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e)); @@ -310,7 +310,7 @@ public override CrossValidationExperimentResult Exe _experiment.SetMonitor(monitor); _experiment.SetTrialRunnerFactory(); - _experiment.SetHyperParameterProposer(); + _experiment.SetHyperParameterProposer(); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => ToCrossValidationRunDetail(e)); @@ -418,7 +418,7 @@ internal class BinaryClassificationCVRunner : ITrialRunner private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; - public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer, SweepablePipeline pipeline) + public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoParameterProposer proposer, SweepablePipeline pipeline) { _context = context; _datasetManager = datasetManager; @@ -477,7 +477,7 @@ internal class BinaryClassificationTrainTestRunner : ITrialRunner private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; - public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciHyperParameterProposer proposer, SweepablePipeline pipeline) + public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoParameterProposer proposer, SweepablePipeline pipeline) { _context = context; _metricManager = metricManager; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 109aa8bd29..99f75c5670 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -40,12 +40,13 @@ private void InitializeServiceCollection() _serviceCollection.TryAddSingleton(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); + _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddScoped(); + _serviceCollection.TryAddScoped(); } private void Initialize() @@ -68,6 +69,7 @@ public AutoMLExperiment SetDataset(IDataView train, IDataView test) }; _serviceCollection.AddSingleton(datasetManager); + _serviceCollection.AddSingleton(datasetManager); return this; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/EciCfoParameterProposer.cs similarity index 58% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/EciCfoParameterProposer.cs index 19eb3c22b9..1ac6b9aa1a 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/HyperParameterProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/EciCfoParameterProposer.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; -using System.Linq; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.SearchSpace; namespace Microsoft.ML.AutoML { @@ -14,16 +14,18 @@ namespace Microsoft.ML.AutoML /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer /// is more likely to be picked. /// - internal class EciHyperParameterProposer : IHyperParameterProposer + internal class EciCfoParameterProposer : IHyperParameterProposer { private readonly Dictionary _tuners; private readonly PipelineProposer _pipelineProposer; // this dictionary records the schema for each trial. // the key is trial id, and value is the schema for that trial. private readonly IMetricManager _metricManager; + private readonly ITuner _rootTuner; - public EciHyperParameterProposer(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) + public EciCfoParameterProposer(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings, ITunerFactory tunerFactory) { + _rootTuner = tunerFactory.CreateTuner(null); _tuners = new Dictionary(); _pipelineProposer = new PipelineProposer(sweepablePipeline, settings, metricManager); _metricManager = metricManager; @@ -31,6 +33,7 @@ public EciHyperParameterProposer(SweepablePipeline sweepablePipeline, IMetricMan public TrialSettings Propose(TrialSettings settings) { + var rootParameter = _rootTuner.Propose(settings); (var searchSpace, var schema) = _pipelineProposer.ProposeSearchSpace(); if (!_tuners.ContainsKey(schema)) { @@ -39,47 +42,23 @@ public TrialSettings Propose(TrialSettings settings) } var tuner = _tuners[schema]; - var parameter = tuner.Propose(settings); + rootParameter[AutoMLExperiment.PipelineSearchspaceName] = tuner.Propose(settings); + var parameter = rootParameter; + settings.Parameter = parameter; return settings; } public void Update(TrialSettings settings, TrialResult result) { - var schema = settings.Parameter["_SCHEMA_"].AsType(); + var schema = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType(); if (_tuners.TryGetValue(schema, out var tuner)) { tuner.Update(result); } _pipelineProposer.Update(settings, result, schema); - } - } - - internal class NestedSearchSpaceHyperParameterProposer : IHyperParameterProposer - { - private readonly ITuner _tuner; - private readonly SweepablePipeline _pipeline; - - public NestedSearchSpaceHyperParameterProposer(SweepablePipeline pipeline, ITunerFactory tunerFactory) - { - this._tuner = tunerFactory.CreateTuner(null); - this._pipeline = pipeline; - } - - public TrialSettings Propose(TrialSettings settings) - { - var parameter = _tuner.Propose(settings); - settings.Parameter = parameter; - var keys = parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType().Replace(" ", string.Empty).Split('*'); - var estimators = keys.Select(k => _pipeline.Estimators[k]); - - return settings; - } - - public void Update(TrialSettings parameter, TrialResult result) - { - _tuner.Update(result); + _rootTuner.Update(result); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs index a29d4d4d28..790366c2a4 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs @@ -8,7 +8,10 @@ namespace Microsoft.ML.AutoML { - internal interface IHyperParameterProposer : ITrialSettingsProposer + internal interface IHyperParameterProposer { + TrialSettings Propose(TrialSettings settings); + + void Update(TrialSettings parameter, TrialResult result); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs new file mode 100644 index 0000000000..5da7438405 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Linq; + +namespace Microsoft.ML.AutoML +{ + internal class NestedSearchSpaceHyperParameterProposer : IHyperParameterProposer + { + private readonly ITuner _tuner; + private readonly SweepablePipeline _pipeline; + + public NestedSearchSpaceHyperParameterProposer(SweepablePipeline pipeline, ITunerFactory tunerFactory) + { + this._tuner = tunerFactory.CreateTuner(null); + this._pipeline = pipeline; + } + + public TrialSettings Propose(TrialSettings settings) + { + var parameter = _tuner.Propose(settings); + settings.Parameter = parameter; + var keys = parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType().Replace(" ", string.Empty).Split('*'); + var estimators = keys.Select(k => _pipeline.Estimators[k]); + + return settings; + } + + public void Update(TrialSettings parameter, TrialResult result) + { + _tuner.Update(result); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/PipelineProposer.cs similarity index 100% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/PipelineProposer.cs diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index 3ea14dd01b..cd9533a606 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -19,7 +19,14 @@ internal interface ICrossValidateDatasetManager IDataView Dataset { get; set; } } - public class TrainTestDatasetManager : IDatasetManager + internal interface ITrainTestDatasetManager + { + IDataView TrainDataset { get; set; } + + IDataView TestDataset { get; set; } + } + + public class TrainTestDatasetManager : IDatasetManager, ITrainTestDatasetManager { public IDataView TrainDataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs new file mode 100644 index 0000000000..54135f2938 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + internal class SweepablePipelineTrainTestRunner : ITrialRunner + { + private readonly MLContext _mLContext; + private readonly IEvaluateMetricManager _metricManager; + private readonly ITrainTestDatasetManager _trainTestDatasetManager; + private readonly SweepablePipeline _pipeline; + + public SweepablePipelineTrainTestRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, ITrainTestDatasetManager trainTestDatasetManager) + { + _mLContext = context; + _metricManager = metricManager; + _pipeline = pipeline; + _trainTestDatasetManager = trainTestDatasetManager; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); + var model = mlnetPipeline.Fit(_trainTestDatasetManager.TrainDataset); + var eval = model.Transform(_trainTestDatasetManager.TestDataset); + var metric = _metricManager.Evaluate(_mLContext, eval); + stopWatch.Stop(); + + return new TrialResult + { + Metric = metric, + Model = model, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + TrialSettings = settings, + }; + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index 7734f0e4e1..19ec800eab 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -49,6 +49,7 @@ public SweepablePipelineTrialRunnerFactory(IServiceProvider provider) ITrialRunner? runner = (datasetManager, metricManager) switch { (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), + (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), _ => throw new NotImplementedException(), }; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs deleted file mode 100644 index ad73331fab..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.ML.AutoML -{ - internal interface ITrialSettingsProposer - { - TrialSettings Propose(TrialSettings settings); - - void Update(TrialSettings parameter, TrialResult result); - } - - internal interface ISavableProposer : ITrialSettingsProposer - { - void SaveStatusToFile(string fileName); - - void LoadStatusFromFile(string fileName); - } -} diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 5e343a3677..52c12f06a5 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -104,6 +104,56 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() cts.IsCancellationRequested.Should().BeFalse(); } + [Fact] + public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetUciAdultDataView(); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) + .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false)); + + experiment.SetDataset(context.Data.TrainTestSplit(data)) + .SetEvaluateMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } + + [Fact] + public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetUciAdultDataView(); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) + .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false)); + + experiment.SetDataset(data, 5) + .SetEvaluateMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } + private IDataView GetDummyData() { var x = Enumerable.Range(-10000, 10000).Select(value => value * 1f).ToArray(); From cd92de597d788310ad15d84eab66b77b5ab10894 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 11:33:45 -0700 Subject: [PATCH 04/25] implement regression && multiclass evaluation metric and add tests --- .../API/SweepableExtension.cs | 5 + .../AutoMLExperiment/AutoMLExperiment.cs | 2 + .../AutoMLExperiment/IMetricManager.cs | 37 +++++- .../AutoMLExperiment/TrialRunnerFactory.cs | 6 +- ...timatorToSweepabePipelineTest.approved.txt | 34 ++++++ .../AutoMLExperimentTests.cs | 111 +++++++++++++++++- test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs | 31 +++++ .../SweepableExtensionTest.cs | 15 +++ 8 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index 0c483392d7..e70f1c826b 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -16,6 +16,11 @@ public static SweepableEstimatorPipeline Append(this SweepableEstimatorPipeline return pipeline.Append(new SweepableEstimator((context, parameter) => estimator1, new SearchSpace.SearchSpace())); } + public static SweepablePipeline Append(this SweepablePipeline pipeline, IEstimator estimator) + { + return pipeline.Append(new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace())); + } + public static SweepablePipeline Append(this SweepableEstimatorPipeline pipeline, SweepablePipeline sweepablePipeline) { var res = new SweepablePipeline(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 99f75c5670..433d80820d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -249,6 +249,7 @@ public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); + _serviceCollection.AddSingleton(metricManager); return this; } @@ -262,6 +263,7 @@ public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelC LabelColumn = labelColumn, }; _serviceCollection.AddSingleton(metricManager); + _serviceCollection.AddSingleton(metricManager); return this; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index df4cb17eb9..e6c11d4496 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -70,7 +70,7 @@ public double Evaluate(MLContext context, IDataView eval) } } - internal class MultiClassMetricManager : IMetricManager + internal class MultiClassMetricManager : IEvaluateMetricManager { public MulticlassClassificationMetric Metric { get; set; } @@ -87,9 +87,26 @@ internal class MultiClassMetricManager : IMetricManager MulticlassClassificationMetric.TopKAccuracy => true, _ => throw new NotImplementedException(), }; + + public string MetricName => Metric.ToString(); + + public double Evaluate(MLContext context, IDataView eval) + { + var metric = context.MulticlassClassification.Evaluate(eval, labelColumnName: LabelColumn, predictedLabelColumnName: PredictedColumn); + + return Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => metric.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => metric.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => metric.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metric.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => metric.TopKAccuracy, + _ => throw new NotImplementedException(), + }; + } } - internal class RegressionMetricManager : IMetricManager + internal class RegressionMetricManager : IEvaluateMetricManager { public RegressionMetric Metric { get; set; } @@ -105,5 +122,21 @@ internal class RegressionMetricManager : IMetricManager RegressionMetric.MeanAbsoluteError => false, _ => throw new NotImplementedException(), }; + + public string MetricName => Metric.ToString(); + + public double Evaluate(MLContext context, IDataView eval) + { + var metric = context.Regression.Evaluate(eval, LabelColumn, ScoreColumn); + + return Metric switch + { + RegressionMetric.RSquared => metric.RSquared, + RegressionMetric.RootMeanSquaredError => metric.RootMeanSquaredError, + RegressionMetric.MeanSquaredError => metric.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => metric.MeanAbsoluteError, + _ => throw new NotImplementedException(), + }; + } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index 19ec800eab..71d257cd22 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -46,10 +46,10 @@ public SweepablePipelineTrialRunnerFactory(IServiceProvider provider) var datasetManager = _provider.GetService(); var metricManager = _provider.GetService(); - ITrialRunner? runner = (datasetManager, metricManager) switch + ITrialRunner? runner = (datasetManager) switch { - (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), + CrossValidateDatasetManager => _provider.GetService(), + TrainTestDatasetManager => _provider.GetService(), _ => throw new NotImplementedException(), }; diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt new file mode 100644 index 0000000000..4a334ab2f5 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt @@ -0,0 +1,34 @@ +{ + "schema": "e0 * (e1 + e2) * e3", + "currentSchema": "e0 * e1 * e3", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e2": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e3": { + "estimatorType": "Unknown", + "parameter": {} + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 52c12f06a5..ded7c0b7ae 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -118,7 +118,7 @@ public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() var data = DatasetUtil.GetUciAdultDataView(); var experiment = context.Auto().CreateExperiment(); var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) - .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false)); + .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(context.Data.TrainTestSplit(data)) .SetEvaluateMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) @@ -143,7 +143,7 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() var data = DatasetUtil.GetUciAdultDataView(); var experiment = context.Auto().CreateExperiment(); var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) - .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false)); + .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(data, 5) .SetEvaluateMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) @@ -154,6 +154,113 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() result.Metric.Should().BeGreaterThan(0.8); } + [Fact] + public async Task AutoMLExperiment_Iris_CV_5_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetIrisDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = "Label"; + var pipeline = context.Auto().Featurizer(data, excludeColumns: new[] { label }) + .Append(context.Transforms.Conversion.MapValueToKey(label, label)) + .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(data, 5) + .SetEvaluateMetric(MulticlassClassificationMetric.MacroAccuracy, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } + + [Fact] + public async Task AutoMLExperiment_Iris_Train_Test_Split_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetIrisDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = "Label"; + var pipeline = context.Auto().Featurizer(data, excludeColumns: new[] { label }) + .Append(context.Transforms.Conversion.MapValueToKey(label, label)) + .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(context.Data.TrainTestSplit(data)) + .SetEvaluateMetric(MulticlassClassificationMetric.MacroAccuracy, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } + + [Fact] + public async Task AutoMLExperiment_Taxi_Fare_Train_Test_Split_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var train = DatasetUtil.GetTaxiFareTrainDataView(); + var test = DatasetUtil.GetTaxiFareTestDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = DatasetUtil.TaxiFareLabel; + var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label }) + .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(train, test) + .SetEvaluateMetric(RegressionMetric.RSquared, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(50); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.5); + } + + [Fact] + public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var train = DatasetUtil.GetTaxiFareTrainDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = DatasetUtil.TaxiFareLabel; + var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label }) + .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(train, 5) + .SetEvaluateMetric(RegressionMetric.RSquared, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(50); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.5); + } + private IDataView GetDummyData() { var x = Enumerable.Range(-10000, 10000).Select(value => value * 1f).ToArray(); diff --git a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs index 17cdfb2791..8c72d86283 100644 --- a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs +++ b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs @@ -16,6 +16,7 @@ namespace Microsoft.ML.AutoML.Test internal static class DatasetUtil { public const string UciAdultLabel = DefaultColumnNames.Label; + public const string TaxiFareLabel = "fare_amount"; public const string TrivialMulticlassDatasetLabel = "Target"; public const string MlNetGeneratedRegressionLabel = "target"; public const int IrisDatasetLabelColIndex = 0; @@ -24,6 +25,10 @@ internal static class DatasetUtil private static IDataView _uciAdultDataView; + private static IDataView _taxiFareTrainDataView; + + private static IDataView _taxiFareTestDataView; + private static IDataView _irisDataView; public static string GetUciAdultDataset() => GetDataPath("adult.tiny.with-schema.txt"); @@ -65,6 +70,32 @@ public static IDataView GetIrisDataView() return _irisDataView; } + public static IDataView GetTaxiFareTrainDataView() + { + if (_taxiFareTrainDataView == null) + { + var context = new MLContext(1); + var taxiFareFile = GetDataPath("taxi-fare-train.csv"); + var columnInferenceResult = context.Auto().InferColumns(taxiFareFile, TaxiFareLabel); + var textLoader = context.Data.CreateTextLoader(columnInferenceResult.TextLoaderOptions); + _taxiFareTrainDataView = textLoader.Load(taxiFareFile); + } + return _taxiFareTrainDataView; + } + + public static IDataView GetTaxiFareTestDataView() + { + if (_taxiFareTestDataView == null) + { + var context = new MLContext(1); + var taxiFareFile = GetDataPath("taxi-fare-test.csv"); + var columnInferenceResult = context.Auto().InferColumns(taxiFareFile, TaxiFareLabel); + var textLoader = context.Data.CreateTextLoader(columnInferenceResult.TextLoaderOptions); + _taxiFareTestDataView = textLoader.Load(taxiFareFile); + } + return _taxiFareTestDataView; + } + public static string GetFlowersDataset() { const string datasetName = @"flowers"; diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index e139061b36..afba4771b9 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -191,5 +191,20 @@ public void CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray() var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); Approvals.Verify(json); } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void AppendIEstimatorToSweepabePipelineTest() + { + var context = new MLContext(); + var estimator = context.Transforms.Concatenate("output", "input"); + var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()), SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); + + pipeline.Should().BeOfType(); + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } } } From 9ea3f430ba11a59b23c3aacbd6ffe4a65cef23fc Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 13:03:30 -0700 Subject: [PATCH 05/25] remove tunarFactory --- .../API/BinaryClassificationExperiment.cs | 196 +++++++----------- .../AutoMLExperiment/AutoMLExperiment.cs | 69 +++--- .../IHyperParameterProposer.cs | 17 -- ...NestedSearchSpaceHyperParameterProposer.cs | 35 ---- .../Runner/SweepablePipelineCVRunner.cs | 57 ----- .../Runner/SweepablePipelineRunner.cs | 79 +++++++ .../SweepablePipelineTrainTestRunner.cs | 49 ----- .../AutoMLExperiment/TrialRunnerFactory.cs | 59 ------ .../EciCfoParameterProposer.cs | 22 +- .../PipelineProposer.cs | 2 +- .../AutoMLExperiment/TunerFactory.cs | 67 ------ 11 files changed, 185 insertions(+), 467 deletions(-) delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs rename src/Microsoft.ML.AutoML/AutoMLExperiment/{HyperParameterProposers => Tuner}/EciCfoParameterProposer.cs (68%) rename src/Microsoft.ML.AutoML/AutoMLExperiment/{HyperParameterProposers => Tuner}/PipelineProposer.cs (99%) delete mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index fccecedf98..d1d97d802f 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -209,8 +209,7 @@ public override ExperimentResult Execute(IDataView progressHandler?.Report(detail); }; - _experiment.SetTrialRunnerFactory(); - _experiment.SetHyperParameterProposer(); + _experiment.SetTrialRunner(); _experiment.SetMonitor(monitor); _experiment.Run(); @@ -248,8 +247,7 @@ public override ExperimentResult Execute(IDataView }; _experiment.SetMonitor(monitor); - _experiment.SetTrialRunnerFactory(); - _experiment.SetHyperParameterProposer(); + _experiment.SetTrialRunner(); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e)); @@ -309,8 +307,7 @@ public override CrossValidationExperimentResult Exe }; _experiment.SetMonitor(monitor); - _experiment.SetTrialRunnerFactory(); - _experiment.SetHyperParameterProposer(); + _experiment.SetTrialRunner(); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => ToCrossValidationRunDetail(e)); @@ -411,14 +408,14 @@ public void ReportRunningTrial(TrialSettings setting) } } - internal class BinaryClassificationCVRunner : ITrialRunner + internal class BinaryClassificationRunner : ITrialRunner { private readonly MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; - public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoParameterProposer proposer, SweepablePipeline pipeline) + public BinaryClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoParameterProposer proposer, SweepablePipeline pipeline) { _context = context; _datasetManager = datasetManager; @@ -428,126 +425,79 @@ public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetMa public TrialResult Run(TrialSettings settings, IServiceProvider provider) { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is BinaryMetricManager metricSettings) + var rnd = settings.ExperimentSettings.Seed.HasValue ? new Random(settings.ExperimentSettings.Seed.Value) : new Random(); + if (_metricManager is BinaryMetricManager metricManager) { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - var pipeline = _pipeline.BuildFromOption(_context, settings.Parameter); - var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var pipeline = _pipeline.BuildFromOption(_context, parameter); + if (_datasetManager is ICrossValidateDatasetManager datasetManager) { - BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new BinaryClassificationTrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - BinaryClassificationMetrics = res.Metrics, - CrossValidationMetrics = metrics, - Pipeline = pipeline, - }; - } - - throw new ArgumentException(); - } - } - - - internal class BinaryClassificationTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - private readonly SweepablePipeline _pipeline; - - public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoParameterProposer proposer, SweepablePipeline pipeline) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - _pipeline = pipeline; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is BinaryMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var pipeline = _pipeline.BuildFromOption(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var metric = metricSettings.Metric switch + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetManager.Fold ?? 5; + var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricManager.Metric switch + { + BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), + }; + + stopWatch.Stop(); + + + return new BinaryClassificationTrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + BinaryClassificationMetrics = res.Metrics, + CrossValidationMetrics = metrics, + Pipeline = pipeline, + }; + } + + if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager) { - BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new BinaryClassificationTrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - BinaryClassificationMetrics = metrics, - Pipeline = pipeline, - }; + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); + var eval = model.Transform(trainTestDatasetManager.TestDataset); + var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var metric = Enum.Parse(typeof(BinaryClassificationMetric), metricManager.MetricName) switch + { + BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new BinaryClassificationTrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + BinaryClassificationMetrics = metrics, + Pipeline = pipeline, + }; + } } - throw new ArgumentException(); - } - } - - - internal class BinaryExperimentTrialRunnerFactory : ITrialRunnerFactory - { - private readonly IServiceProvider _provider; - - public BinaryExperimentTrialRunnerFactory(IServiceProvider provider) - { - _provider = provider; - } - - public ITrialRunner CreateTrialRunner() - { - var datasetManager = _provider.GetService(); - var metricManager = _provider.GetService(); - - ITrialRunner runner = (datasetManager, metricManager) switch - { - (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), - _ => throw new NotImplementedException(), - }; - - return runner; + throw new ArgumentException("IMetricManager must be BinaryMetricManager and IDatasetManager must be either TrainTestSplitDatasetManager or CrossValidationDatasetManager"); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 433d80820d..b3b8d00a65 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ML.Runtime; +using Microsoft.ML.SearchSpace; using static Microsoft.ML.DataOperationsCatalog; namespace Microsoft.ML.AutoML @@ -37,16 +38,8 @@ private void InitializeServiceCollection() { _serviceCollection.TryAddSingleton(_context); _serviceCollection.TryAddSingleton(_settings); - _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddScoped(); + _serviceCollection.TryAddSingleton(); + _serviceCollection.TryAddSingleton(); } private void Initialize() @@ -95,22 +88,6 @@ public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) return this; } - public AutoMLExperiment SetTunerFactory() - where TTunerFactory : ITunerFactory - { - var descriptor = new ServiceDescriptor(typeof(ITunerFactory), typeof(TTunerFactory), ServiceLifetime.Singleton); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } - - return this; - } - public AutoMLExperiment SetMonitor(TMonitor monitor) where TMonitor : class, IMonitor { @@ -147,15 +124,15 @@ internal AutoMLExperiment SetPipeline(SweepablePipeline pipeline) _settings.SearchSpace[PipelineSearchspaceName] = pipeline.SearchSpace; _serviceCollection.AddSingleton(pipeline); - SetTrialRunnerFactory(); + SetTrialRunner(); SetMonitor(); return this; } - public AutoMLExperiment SetTrialRunner(ITrialRunner runner) + public AutoMLExperiment SetTrialRunner(TTrialRunner runner) + where TTrialRunner : class, ITrialRunner { - var factory = new CustomRunnerFactory(runner); - var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), factory); + var descriptor = new ServiceDescriptor(typeof(ITrialRunner), runner); if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -168,10 +145,10 @@ public AutoMLExperiment SetTrialRunner(ITrialRunner runner) return this; } - internal AutoMLExperiment SetTrialRunnerFactory() - where TTrialRunnerFactory : ITrialRunnerFactory + public AutoMLExperiment SetTrialRunner() + where TTrialRunner : class, ITrialRunner { - var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), typeof(TTrialRunnerFactory), ServiceLifetime.Singleton); + var descriptor = new ServiceDescriptor(typeof(ITrialRunner), typeof(TTrialRunner), ServiceLifetime.Transient); if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -184,11 +161,12 @@ internal AutoMLExperiment SetTrialRunnerFactory() return this; } - internal AutoMLExperiment SetHyperParameterProposer(THyperParameterProposer proposer) - where THyperParameterProposer : class, IHyperParameterProposer + internal AutoMLExperiment SetTuner(TTuner proposer) + where TTuner : class, ITuner { _serviceCollection.TryAddSingleton(proposer); - var descriptor = new ServiceDescriptor(typeof(IHyperParameterProposer), proposer); + var descriptor = new ServiceDescriptor(typeof(ITuner), proposer); + if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -201,11 +179,11 @@ internal AutoMLExperiment SetHyperParameterProposer(THy return this; } - internal AutoMLExperiment SetHyperParameterProposer() - where THyperParameterProposer : class, IHyperParameterProposer + internal AutoMLExperiment SetTuner() + where TTuner : class, ITuner { - _serviceCollection.TryAddSingleton(); - var descriptor = new ServiceDescriptor(typeof(IHyperParameterProposer), typeof(THyperParameterProposer), ServiceLifetime.Singleton); + _serviceCollection.TryAddSingleton(); + var descriptor = new ServiceDescriptor(typeof(ITuner), typeof(TTuner), ServiceLifetime.Singleton); if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -302,8 +280,7 @@ public async Task RunAsync(CancellationToken ct = default) var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); var trialNum = 0; - var hyperParameterProposer = serviceProvider.GetService(); - var runnerFactory = serviceProvider.GetService(); + var tuner = serviceProvider.GetService(); var metricManager = serviceProvider.GetService(); while (true) @@ -316,17 +293,19 @@ public async Task RunAsync(CancellationToken ct = default) { ExperimentSettings = _settings, TrialId = trialNum++, + Parameter = Parameter.CreateNestedParameter(), }; - setting = hyperParameterProposer.Propose(setting); + var parameter = tuner.Propose(setting); + setting.Parameter = parameter; monitor.ReportRunningTrial(setting); - var runner = runnerFactory.CreateTrialRunner(); + var runner = serviceProvider.GetService(); try { var trialResult = runner.Run(setting, serviceProvider); monitor.ReportCompletedTrial(trialResult); - hyperParameterProposer.Update(setting, trialResult); + tuner.Update(trialResult); var error = metricManager.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs deleted file mode 100644 index 790366c2a4..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/IHyperParameterProposer.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace Microsoft.ML.AutoML -{ - internal interface IHyperParameterProposer - { - TrialSettings Propose(TrialSettings settings); - - void Update(TrialSettings parameter, TrialResult result); - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs deleted file mode 100644 index 5da7438405..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/NestedSearchSpaceHyperParameterProposer.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Linq; - -namespace Microsoft.ML.AutoML -{ - internal class NestedSearchSpaceHyperParameterProposer : IHyperParameterProposer - { - private readonly ITuner _tuner; - private readonly SweepablePipeline _pipeline; - - public NestedSearchSpaceHyperParameterProposer(SweepablePipeline pipeline, ITunerFactory tunerFactory) - { - this._tuner = tunerFactory.CreateTuner(null); - this._pipeline = pipeline; - } - - public TrialSettings Propose(TrialSettings settings) - { - var parameter = _tuner.Propose(settings); - settings.Parameter = parameter; - var keys = parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType().Replace(" ", string.Empty).Split('*'); - var estimators = keys.Select(k => _pipeline.Estimators[k]); - - return settings; - } - - public void Update(TrialSettings parameter, TrialResult result) - { - _tuner.Update(result); - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs deleted file mode 100644 index fe46331d51..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineCVRunner.cs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using Microsoft.ML.Data; - -namespace Microsoft.ML.AutoML -{ - internal class SweepablePipelineCVRunner : ITrialRunner - { - private readonly MLContext _mLContext; - private readonly IEvaluateMetricManager _metricManager; - private readonly ICrossValidateDatasetManager _crossValidateDatasetManager; - private readonly SweepablePipeline _pipeline; - - public SweepablePipelineCVRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, ICrossValidateDatasetManager crossValidateDatasetManager) - { - _mLContext = context; - _metricManager = metricManager; - _pipeline = pipeline; - _crossValidateDatasetManager = crossValidateDatasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; - var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); - var datasetSplit = _mLContext.Data.CrossValidationSplit(_crossValidateDatasetManager.Dataset, _crossValidateDatasetManager.Fold ?? 5); - var metrics = new List(); - var models = new List(); - foreach (var split in datasetSplit) - { - var model = mlnetPipeline.Fit(split.TrainSet); - var eval = model.Transform(split.TestSet); - metrics.Add(_metricManager.Evaluate(_mLContext, eval)); - models.Add(model); - } - - stopWatch.Stop(); - - return new TrialResult - { - Metric = metrics.Average(), - Model = models.First(), - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - TrialSettings = settings, - }; - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs new file mode 100644 index 0000000000..d9c9f44f68 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + internal class SweepablePipelineRunner : ITrialRunner + { + private readonly MLContext _mLContext; + private readonly IEvaluateMetricManager _metricManager; + private readonly IDatasetManager _datasetManager; + private readonly SweepablePipeline _pipeline; + + public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager) + { + _mLContext = context; + _metricManager = metricManager; + _pipeline = pipeline; + _datasetManager = datasetManager; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); + + if (_datasetManager is ICrossValidateDatasetManager crossValidateDatasetManager) + { + var datasetSplit = _mLContext.Data.CrossValidationSplit(crossValidateDatasetManager.Dataset, crossValidateDatasetManager.Fold ?? 5); + var metrics = new List(); + var models = new List(); + foreach (var split in datasetSplit) + { + var model = mlnetPipeline.Fit(split.TrainSet); + var eval = model.Transform(split.TestSet); + metrics.Add(_metricManager.Evaluate(_mLContext, eval)); + models.Add(model); + } + + stopWatch.Stop(); + + return new TrialResult + { + Metric = metrics.Average(), + Model = models.First(), + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + TrialSettings = settings, + }; + } + + if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager) + { + var model = mlnetPipeline.Fit(trainTestDatasetManager.TrainDataset); + var eval = model.Transform(trainTestDatasetManager.TestDataset); + var metric = _metricManager.Evaluate(_mLContext, eval); + stopWatch.Stop(); + + return new TrialResult + { + Metric = metric, + Model = model, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + TrialSettings = settings, + }; + } + + throw new ArgumentException("IDatasetManager must be either ITrainTestDatasetManager or ICrossValidationDatasetManager"); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs deleted file mode 100644 index 54135f2938..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineTrainTestRunner.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using Microsoft.ML.Data; - -namespace Microsoft.ML.AutoML -{ - internal class SweepablePipelineTrainTestRunner : ITrialRunner - { - private readonly MLContext _mLContext; - private readonly IEvaluateMetricManager _metricManager; - private readonly ITrainTestDatasetManager _trainTestDatasetManager; - private readonly SweepablePipeline _pipeline; - - public SweepablePipelineTrainTestRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, ITrainTestDatasetManager trainTestDatasetManager) - { - _mLContext = context; - _metricManager = metricManager; - _pipeline = pipeline; - _trainTestDatasetManager = trainTestDatasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; - var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); - var model = mlnetPipeline.Fit(_trainTestDatasetManager.TrainDataset); - var eval = model.Transform(_trainTestDatasetManager.TestDataset); - var metric = _metricManager.Evaluate(_mLContext, eval); - stopWatch.Stop(); - - return new TrialResult - { - Metric = metric, - Model = model, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - TrialSettings = settings, - }; - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs deleted file mode 100644 index 71d257cd22..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using Microsoft.Extensions.DependencyInjection; - -#nullable enable -namespace Microsoft.ML.AutoML -{ - /// - /// interface for trial runner factory. - /// - public interface ITrialRunnerFactory - { - ITrialRunner? CreateTrialRunner(); - } - - internal class CustomRunnerFactory : ITrialRunnerFactory - { - private readonly ITrialRunner _instance; - - public CustomRunnerFactory(ITrialRunner runner) - { - _instance = runner; - } - - public ITrialRunner? CreateTrialRunner() - { - return _instance; - } - } - - - internal class SweepablePipelineTrialRunnerFactory : ITrialRunnerFactory - { - private readonly IServiceProvider _provider; - - public SweepablePipelineTrialRunnerFactory(IServiceProvider provider) - { - _provider = provider; - } - - public ITrialRunner? CreateTrialRunner() - { - var datasetManager = _provider.GetService(); - var metricManager = _provider.GetService(); - - ITrialRunner? runner = (datasetManager) switch - { - CrossValidateDatasetManager => _provider.GetService(), - TrainTestDatasetManager => _provider.GetService(), - _ => throw new NotImplementedException(), - }; - - return runner; - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/EciCfoParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/EciCfoParameterProposer.cs similarity index 68% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/EciCfoParameterProposer.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/EciCfoParameterProposer.cs index 1ac6b9aa1a..06dc0b01a2 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/EciCfoParameterProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/EciCfoParameterProposer.cs @@ -14,26 +14,23 @@ namespace Microsoft.ML.AutoML /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer /// is more likely to be picked. /// - internal class EciCfoParameterProposer : IHyperParameterProposer + internal class EciCfoParameterProposer : ITuner { private readonly Dictionary _tuners; private readonly PipelineProposer _pipelineProposer; // this dictionary records the schema for each trial. // the key is trial id, and value is the schema for that trial. private readonly IMetricManager _metricManager; - private readonly ITuner _rootTuner; - public EciCfoParameterProposer(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings, ITunerFactory tunerFactory) + public EciCfoParameterProposer(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) { - _rootTuner = tunerFactory.CreateTuner(null); _tuners = new Dictionary(); _pipelineProposer = new PipelineProposer(sweepablePipeline, settings, metricManager); _metricManager = metricManager; } - public TrialSettings Propose(TrialSettings settings) + public Parameter Propose(TrialSettings settings) { - var rootParameter = _rootTuner.Propose(settings); (var searchSpace, var schema) = _pipelineProposer.ProposeSearchSpace(); if (!_tuners.ContainsKey(schema)) { @@ -42,23 +39,20 @@ public TrialSettings Propose(TrialSettings settings) } var tuner = _tuners[schema]; - rootParameter[AutoMLExperiment.PipelineSearchspaceName] = tuner.Propose(settings); - var parameter = rootParameter; + settings.Parameter[AutoMLExperiment.PipelineSearchspaceName] = tuner.Propose(settings); - settings.Parameter = parameter; - return settings; + return settings.Parameter; } - public void Update(TrialSettings settings, TrialResult result) + public void Update(TrialResult result) { - var schema = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType(); + var schema = result.TrialSettings.Parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType(); if (_tuners.TryGetValue(schema, out var tuner)) { tuner.Update(result); } - _pipelineProposer.Update(settings, result, schema); - _rootTuner.Update(result); + _pipelineProposer.Update(result, schema); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/PipelineProposer.cs similarity index 99% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/PipelineProposer.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/PipelineProposer.cs index 79ff2dcf78..5773303cd8 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/HyperParameterProposers/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/PipelineProposer.cs @@ -124,7 +124,7 @@ public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSet return (_sweepablePipeline.BuildSweepableEstimatorPipeline(schema).SearchSpace, schema); } - public void Update(TrialSettings parameter, TrialResult result, string schema) + public void Update(TrialResult result, string schema) { var error = CaculateError(result.Metric, _metricManager.IsMaximize); var duration = result.DurationInMilliseconds / 1000; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs deleted file mode 100644 index 3412143eea..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using Microsoft.Extensions.DependencyInjection; - -namespace Microsoft.ML.AutoML -{ - /// - /// interface for all tuner factories. - /// - public interface ITunerFactory - { - ITuner CreateTuner(TrialSettings settings); - } - - internal class CostFrugalTunerFactory : ITunerFactory - { - private readonly SearchSpace.SearchSpace _searchSpace; - private readonly IMetricManager _metricManager; - - public CostFrugalTunerFactory(AutoMLExperiment.AutoMLExperimentSettings settings, IMetricManager metricManager) - { - _searchSpace = settings.SearchSpace; - _metricManager = metricManager; - } - - public ITuner CreateTuner(TrialSettings settings) - { - var initParameter = _searchSpace.SampleFromFeatureSpace(_searchSpace.Default); - var isMaximize = _metricManager.IsMaximize; - - return new CostFrugalTuner(_searchSpace, initParameter, !isMaximize); - } - } - - internal class RandomTunerFactory : ITunerFactory - { - private readonly SearchSpace.SearchSpace _searchSpace; - - public RandomTunerFactory(AutoMLExperiment.AutoMLExperimentSettings settings) - { - _searchSpace = settings.SearchSpace; - } - - public ITuner CreateTuner(TrialSettings settings) - { - return new RandomSearchTuner(_searchSpace); - } - } - - internal class GridSearchTunerFactory : ITunerFactory - { - private readonly SearchSpace.SearchSpace _searchSpace; - - public GridSearchTunerFactory(AutoMLExperiment.AutoMLExperimentSettings settings) - { - _searchSpace = settings.SearchSpace; - } - - public ITuner CreateTuner(TrialSettings settings) - { - return new GridSearchTuner(_searchSpace); - } - } -} From 0c1d0d43eb7d88e01ce13942f9f5e48b8b1441aa Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 13:20:10 -0700 Subject: [PATCH 06/25] rename EciCfoParameterProposer to EciCfoTuner --- .../API/BinaryClassificationExperiment.cs | 2 +- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 4 ++-- src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs | 4 ++-- src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs | 5 +++++ .../EciCfoParameterProposer.cs => Tuner/EciCfoTuner.cs} | 4 ++-- src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs | 5 +++++ .../{AutoMLExperiment => }/Tuner/PipelineProposer.cs | 0 src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs | 5 +++++ 8 files changed, 22 insertions(+), 7 deletions(-) rename src/Microsoft.ML.AutoML/{AutoMLExperiment/Tuner/EciCfoParameterProposer.cs => Tuner/EciCfoTuner.cs} (90%) rename src/Microsoft.ML.AutoML/{AutoMLExperiment => }/Tuner/PipelineProposer.cs (100%) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index d1d97d802f..9ad8fa81b1 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -415,7 +415,7 @@ internal class BinaryClassificationRunner : ITrialRunner private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; - public BinaryClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoParameterProposer proposer, SweepablePipeline pipeline) + public BinaryClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoTuner proposer, SweepablePipeline pipeline) { _context = context; _datasetManager = datasetManager; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index b3b8d00a65..4d94eda03d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -38,8 +38,8 @@ private void InitializeServiceCollection() { _serviceCollection.TryAddSingleton(_context); _serviceCollection.TryAddSingleton(_settings); - _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddSingleton(); + _serviceCollection.TryAddSingleton(); + _serviceCollection.TryAddSingleton(); } private void Initialize() diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index e6c11d4496..8c4d6ab9bc 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -9,12 +9,12 @@ namespace Microsoft.ML.AutoML /// /// Interface for metric manager. /// - internal interface IMetricManager + public interface IMetricManager { bool IsMaximize { get; } } - internal interface IEvaluateMetricManager : IMetricManager + public interface IEvaluateMetricManager : IMetricManager { double Evaluate(MLContext context, IDataView eval); diff --git a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs index 949f851dff..5622c0c1ed 100644 --- a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs @@ -25,6 +25,11 @@ internal class CostFrugalTuner : ITuner private bool _initUsed = false; private double _bestMetric; + public CostFrugalTuner(AutoMLExperiment.AutoMLExperimentSettings settings, IMetricManager manager) + : this(settings.SearchSpace, settings.SearchSpace.SampleFromFeatureSpace(settings.SearchSpace.Default), !manager.IsMaximize) + { + } + public CostFrugalTuner(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true) { _searchSpace = searchSpace; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/EciCfoParameterProposer.cs b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs similarity index 90% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/EciCfoParameterProposer.cs rename to src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs index 06dc0b01a2..4b38926e64 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/EciCfoParameterProposer.cs +++ b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.AutoML /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer /// is more likely to be picked. /// - internal class EciCfoParameterProposer : ITuner + public class EciCfoTuner : ITuner { private readonly Dictionary _tuners; private readonly PipelineProposer _pipelineProposer; @@ -22,7 +22,7 @@ internal class EciCfoParameterProposer : ITuner // the key is trial id, and value is the schema for that trial. private readonly IMetricManager _metricManager; - public EciCfoParameterProposer(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) + public EciCfoTuner(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) { _tuners = new Dictionary(); _pipelineProposer = new PipelineProposer(sweepablePipeline, settings, metricManager); diff --git a/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs index d060ac9ed7..e68461899b 100644 --- a/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs @@ -12,6 +12,11 @@ internal class GridSearchTuner : ITuner private readonly SearchSpace.Tuner.GridSearchTuner _tuner; private IEnumerator _enumerator; + public GridSearchTuner(AutoMLExperiment.AutoMLExperimentSettings settings) + : this(settings.SearchSpace, 10) + { + } + public GridSearchTuner(SearchSpace.SearchSpace searchSpace, int stepSize = 10) { _tuner = new SearchSpace.Tuner.GridSearchTuner(searchSpace, stepSize); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/PipelineProposer.cs b/src/Microsoft.ML.AutoML/Tuner/PipelineProposer.cs similarity index 100% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/Tuner/PipelineProposer.cs rename to src/Microsoft.ML.AutoML/Tuner/PipelineProposer.cs diff --git a/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs index 0a860a4730..fd63a51dfa 100644 --- a/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs @@ -12,6 +12,11 @@ internal class RandomSearchTuner : ITuner private readonly RandomTuner _tuner; private readonly SearchSpace.SearchSpace _searchSpace; + public RandomSearchTuner(AutoMLExperiment.AutoMLExperimentSettings settings) + : this(settings.SearchSpace) + { + } + public RandomSearchTuner(SearchSpace.SearchSpace searchSpace) { _tuner = new RandomTuner(); From bbe4858e8989894038906d947dfa475e97fb70ff Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 14:28:39 -0700 Subject: [PATCH 07/25] some clean up --- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 4 ---- src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 4d94eda03d..ac8db78281 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -62,7 +62,6 @@ public AutoMLExperiment SetDataset(IDataView train, IDataView test) }; _serviceCollection.AddSingleton(datasetManager); - _serviceCollection.AddSingleton(datasetManager); return this; } @@ -83,7 +82,6 @@ public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) }; _serviceCollection.AddSingleton(datasetManager); - _serviceCollection.AddSingleton(datasetManager); return this; } @@ -164,7 +162,6 @@ public AutoMLExperiment SetTrialRunner() internal AutoMLExperiment SetTuner(TTuner proposer) where TTuner : class, ITuner { - _serviceCollection.TryAddSingleton(proposer); var descriptor = new ServiceDescriptor(typeof(ITuner), proposer); if (_serviceCollection.Contains(descriptor)) @@ -182,7 +179,6 @@ internal AutoMLExperiment SetTuner(TTuner proposer) internal AutoMLExperiment SetTuner() where TTuner : class, ITuner { - _serviceCollection.TryAddSingleton(); var descriptor = new ServiceDescriptor(typeof(ITuner), typeof(TTuner), ServiceLifetime.Singleton); if (_serviceCollection.Contains(descriptor)) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 5ffdd396fb..09f1b66e65 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -26,14 +26,12 @@ public interface IMonitor internal class MLContextMonitor : IMonitor { private readonly MLContext _context; - private readonly IServiceProvider _serviceProvider; private readonly IChannel _logger; private readonly List _completedTrials; private readonly SweepablePipeline _pipeline; - public MLContextMonitor(MLContext context, SweepablePipeline pipeline, IServiceProvider provider) + public MLContextMonitor(MLContext context, SweepablePipeline pipeline) { _context = context; - _serviceProvider = provider; _logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); _completedTrials = new List(); _pipeline = pipeline; From 9b41c1a01a10a204f020c4324c0861bbe160663e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 15:31:58 -0700 Subject: [PATCH 08/25] remove AutoMLExperimentSetting from TrialSetting --- .../NotebookMonitor.cs | 2 +- .../API/BinaryClassificationExperiment.cs | 163 +++----------- .../API/MulticlassClassificationExperiment.cs | 141 ++++++++++-- .../AutoMLExperiment/AutoMLExperiment.cs | 1 - .../AutoMLExperiment/IMonitor.cs | 60 ++++- .../AutoMLExperiment/Runner/TrialRunner.cs | 212 ------------------ .../AutoMLExperiment/TrialResult.cs | 11 +- .../AutoMLExperiment/TrialSettings.cs | 4 - .../Utils/BestResultUtil.cs | 16 +- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 3 +- .../AutoMLExperimentTests.cs | 11 +- 11 files changed, 227 insertions(+), 397 deletions(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index cd555ec14d..81b22a9435 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -17,7 +17,7 @@ public class NotebookMonitor : IMonitor private readonly ActionThrottler _updateThrottler; private DisplayedValue _valueToUpdate; - public SweepablePipeline SweepablePipeline { get; private set;} + public SweepablePipeline SweepablePipeline { get; private set; } public TrialResult BestTrial { get; set; } public TrialResult MostRecentTrial { get; set; } public TrialSettings ActiveTrial { get; set; } diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index ca96da98c9..8854ab53fb 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -168,24 +168,13 @@ public override ExperimentResult Execute(IDataView var splitData = Context.Data.TrainTestSplit(trainData); _experiment.SetDataset(splitData.TrainSet, splitData.TestSet); } - - if (preFeaturizer != null) - { - _pipeline = preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) - .Append(Context.Auto().BinaryClassification(label, Features)); - } - else - { - _pipeline = Context.Auto().Featurizer(trainData, columnInformation, Features) - .Append(Context.Auto().BinaryClassification(label, Features)); - } - + _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context); + var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => { - var detail = BestResultUtil.ToRunDetail(Context, e); + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); progressHandler?.Report(detail); }; @@ -193,8 +182,8 @@ public override ExperimentResult Execute(IDataView _experiment.SetMonitor(monitor); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -206,23 +195,12 @@ public override ExperimentResult Execute(IDataView _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); - - if (preFeaturizer != null) - { - _pipeline = preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) - .Append(Context.Auto().BinaryClassification(label, Features)); - } - else - { - _pipeline = Context.Auto().Featurizer(trainData, columnInformation, Features) - .Append(Context.Auto().BinaryClassification(label, Features)); - } - + _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - var monitor = new BinaryClassificationTrialResultMonitor(); + var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => { - var detail = BestResultUtil.ToRunDetail(Context, e); + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); progressHandler?.Report(detail); }; @@ -230,8 +208,8 @@ public override ExperimentResult Execute(IDataView _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -264,24 +242,13 @@ public override CrossValidationExperimentResult Exe _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); - - if (preFeaturizer != null) - { - _pipeline = preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) - .Append(Context.Auto().BinaryClassification(label, Features)); - } - else - { - _pipeline = Context.Auto().Featurizer(trainData, columnInformation, Features) - .Append(Context.Auto().BinaryClassification(label, Features)); - } - + _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context); + var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => { - var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e); + var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); progressHandler?.Report(runDetails); }; @@ -290,8 +257,8 @@ public override CrossValidationExperimentResult Exe _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e)); - var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline)); + var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun, _pipeline); var result = new CrossValidationExperimentResult(runDetails, bestResult); @@ -319,73 +286,19 @@ private protected override CrossValidationRunDetail return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing); } - private MultiModelPipeline CreateBinaryClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) + private SweepablePipeline CreateBinaryClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { - var trainerName = _pipeline.ToString(result.TrialSettings.Parameter); - var modelContainer = new ModelContainer(Context, result.Model); - return new RunDetail(trainerName, result.Pipeline, null, modelContainer, result.BinaryClassificationMetrics, result.Exception); - } - - private CrossValidationRunDetail ToCrossValidationRunDetail(BinaryClassificationTrialResult result) - { - var trainerName = _pipeline.ToString(result.TrialSettings.Parameter); - var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult(new ModelContainer(Context, m.Model), m.Metrics, result.Exception)); - return new CrossValidationRunDetail(trainerName, result.Pipeline, null, crossValidationResult); - } - } - - internal class BinaryClassificationTrialResultMonitor : IMonitor - { - public BinaryClassificationTrialResultMonitor() - { - this.RunDetails = new List(); - } - - public event EventHandler OnTrialCompleted; - - public List RunDetails { get; } - - public BinaryClassificationTrialResult BestRun { get; private set; } - - public void ReportBestTrial(TrialResult result) - { - if (result is BinaryClassificationTrialResult binaryClassificationResult) - { - BestRun = binaryClassificationResult; - } - else - { - throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}"); - } - } - - public void ReportCompletedTrial(TrialResult result) - { - if (result is BinaryClassificationTrialResult binaryClassificationResult) + if (preFeaturizer != null) { - RunDetails.Add(binaryClassificationResult); - OnTrialCompleted?.Invoke(this, binaryClassificationResult); + return preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) + .Append(Context.Auto().BinaryClassification(columnInformation.LabelColumnName, Features)); } else { - throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}"); + return Context.Auto().Featurizer(trainData, columnInformation, Features) + .Append(Context.Auto().BinaryClassification(columnInformation.LabelColumnName, Features)); } } - - public void ReportFailTrial(TrialSettings settings, Exception exp) - { - var result = new BinaryClassificationTrialResult - { - TrialSettings = settings, - Exception = exp, - }; - - RunDetails.Add(result); - } - - public void ReportRunningTrial(TrialSettings setting) - { - } } internal class BinaryClassificationRunner : ITrialRunner @@ -394,18 +307,18 @@ internal class BinaryClassificationRunner : ITrialRunner private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; - - public BinaryClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, EciCfoTuner proposer, SweepablePipeline pipeline) + private readonly Random _rnd; + public BinaryClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, SweepablePipeline pipeline, AutoMLExperiment.AutoMLExperimentSettings settings) { _context = context; _datasetManager = datasetManager; _metricManager = metricManager; _pipeline = pipeline; + _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); } public TrialResult Run(TrialSettings settings, IServiceProvider provider) { - var rnd = settings.ExperimentSettings.Seed.HasValue ? new Random(settings.ExperimentSettings.Seed.Value) : new Random(); if (_metricManager is BinaryMetricManager metricManager) { var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; @@ -418,7 +331,7 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; + var res = metrics[_rnd.Next(fold)]; var model = res.Model; var metric = metricManager.Metric switch { @@ -432,13 +345,13 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) stopWatch.Stop(); - return new BinaryClassificationTrialResult() + return new TrialResult() { Metric = metric, Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - BinaryClassificationMetrics = res.Metrics, + Metrics = res.Metrics, CrossValidationMetrics = metrics, Pipeline = pipeline, }; @@ -465,13 +378,13 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) stopWatch.Stop(); - return new BinaryClassificationTrialResult() + return new TrialResult() { Metric = metric, Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - BinaryClassificationMetrics = metrics, + Metrics = metrics, Pipeline = pipeline, }; } @@ -480,22 +393,4 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) throw new ArgumentException("IMetricManager must be BinaryMetricManager and IDatasetManager must be either TrainTestSplitDatasetManager or CrossValidationDatasetManager"); } } - - /// - /// TrialResult with Binary Classification Metrics - /// - internal class BinaryClassificationTrialResult : TrialResult - { - public BinaryClassificationMetrics BinaryClassificationMetrics { get; set; } - - public IEnumerable> CrossValidationMetrics { get; set; } - - public Exception Exception { get; set; } - - public bool IsSucceed { get => Exception == null; } - - public bool IsCrossValidation { get => CrossValidationMetrics == null; } - - public EstimatorChain Pipeline { get; set; } - } } diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 7bb7c9a92e..2fb3b8e990 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.Trainers; @@ -121,6 +122,7 @@ public sealed class MulticlassClassificationExperiment : ExperimentBase Execute(IDataV return this.Execute(splitData.TrainSet, splitData.TestSet, columnInformation, preFeaturizer, progressHandler); } - var pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); + _pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context); + var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => { - var detail = BestResultUtil.ToRunDetail(Context, e); + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); progressHandler?.Report(detail); }; + _experiment.SetTrialRunner(); _experiment.SetMonitor(monitor); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -184,20 +187,21 @@ public override ExperimentResult Execute(IDataV _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); - var pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); - var monitor = new TrialResultMonitor(Context); + _pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); + var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => { - var detail = BestResultUtil.ToRunDetail(Context, e); + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); progressHandler?.Report(detail); }; + _experiment.SetTrialRunner(); _experiment.SetMonitor(monitor); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -231,22 +235,23 @@ public override CrossValidationExperimentResult _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); - var pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); + _pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context); + var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => { - var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e); + var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); progressHandler?.Report(runDetails); }; + _experiment.SetTrialRunner(); _experiment.SetMonitor(monitor); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e)); - var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline)); + var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun, _pipeline); var result = new CrossValidationExperimentResult(runDetails, bestResult); @@ -274,7 +279,7 @@ private protected override RunDetail GetBestRun return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing); } - private MultiModelPipeline CreateMulticlassClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) + private SweepablePipeline CreateMulticlassClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { var useSdca = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.SdcaMaximumEntropy); var uselbfgs = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); @@ -282,7 +287,7 @@ private MultiModelPipeline CreateMulticlassClassificationPipeline(IDataView trai var useFastForest = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.FastForestOva); var useFastTree = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.FastTreeOva); - MultiModelPipeline pipeline = new MultiModelPipeline(); + SweepablePipeline pipeline = new SweepablePipeline(); if (preFeaturizer != null) { pipeline = pipeline.Append(preFeaturizer); @@ -298,4 +303,100 @@ private MultiModelPipeline CreateMulticlassClassificationPipeline(IDataView trai return pipeline; } } + + + internal class MulticlassClassificationRunner : ITrialRunner + { + private readonly MLContext _context; + private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; + private readonly SweepablePipeline _pipeline; + private readonly Random _rnd; + + public MulticlassClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, SweepablePipeline pipeline, AutoMLExperiment.AutoMLExperimentSettings settings) + { + _context = context; + _datasetManager = datasetManager; + _metricManager = metricManager; + _pipeline = pipeline; + _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider) + { + if (_metricManager is MultiClassMetricManager metricManager) + { + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var pipeline = _pipeline.BuildFromOption(_context, parameter); + if (_datasetManager is ICrossValidateDatasetManager datasetManager) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetManager.Fold ?? 5; + var metrics = _context.MulticlassClassification.CrossValidate(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[_rnd.Next(fold)]; + var model = res.Model; + var metric = metricManager.Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, + _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + Metrics = res.Metrics, + CrossValidationMetrics = metrics, + Pipeline = pipeline, + }; + } + + if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); + var eval = model.Transform(trainTestDatasetManager.TestDataset); + var metrics = _context.MulticlassClassification.Evaluate(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn); + + var metric = metricManager.Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, + _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + Metrics = metrics, + Pipeline = pipeline, + }; + } + } + + throw new ArgumentException("IMetricManager must be MultiMetricManager and IDatasetManager must be either TrainTestSplitDatasetManager or CrossValidationDatasetManager"); + } + } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index ac8db78281..9ff4e5a8f4 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -287,7 +287,6 @@ public async Task RunAsync(CancellationToken ct = default) } var setting = new TrialSettings() { - ExperimentSettings = _settings, TrialId = trialNum++, Parameter = Parameter.CreateNestedParameter(), }; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 4d9faae994..2ba3efacd7 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -48,14 +48,70 @@ public virtual void ReportCompletedTrial(TrialResult result) _completedTrials.Add(result); } - public void ReportFailTrial(TrialSettings settings, Exception exception = null) + public virtual void ReportFailTrial(TrialSettings settings, Exception exception = null) { _logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {_pipeline.ToString(settings.Parameter)}"); } - public void ReportRunningTrial(TrialSettings setting) + public virtual void ReportRunningTrial(TrialSettings setting) { _logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {_pipeline.ToString(setting.Parameter)}"); } } + + internal class TrialResultMonitor : MLContextMonitor + where TMetrics : class + { + public TrialResultMonitor(MLContext context, SweepablePipeline pipeline) + : base(context, pipeline) + { + this.RunDetails = new List>(); + } + + public event EventHandler> OnTrialCompleted; + + public List> RunDetails { get; } + + public TrialResult BestRun { get; private set; } + + public override void ReportBestTrial(TrialResult result) + { + base.ReportBestTrial(result); + if (result is TrialResult binaryClassificationResult) + { + BestRun = binaryClassificationResult; + } + else + { + throw new ArgumentException($"result must be of type {typeof(TrialResult)}"); + } + } + + public override void ReportCompletedTrial(TrialResult result) + { + base.ReportCompletedTrial(result); + if (result is TrialResult metricResult) + { + RunDetails.Add(metricResult); + OnTrialCompleted?.Invoke(this, metricResult); + } + else + { + throw new ArgumentException($"result must be of type {typeof(TrialResult)}"); + } + } + + public override void ReportFailTrial(TrialSettings settings, Exception exp) + { + base.ReportFailTrial(settings, exp); + + var result = new TrialResult + { + TrialSettings = settings, + Exception = exp, + }; + + RunDetails.Add(result); + } + } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs index a14f33e829..e43d018708 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs @@ -16,216 +16,4 @@ public interface ITrialRunner { TrialResult Run(TrialSettings settings, IServiceProvider provider = null); } - - - internal class MultiClassificationTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public MultiClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is MultiClassMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.MulticlassClassification.Evaluate(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); - - var metric = metricSettings.Metric switch - { - MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, - MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, - MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, - MulticlassClassificationMetric.LogLoss => metrics.LogLoss, - MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new TrialResult() - { - Metrics = metrics, - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - } - - throw new ArgumentException(); - } - } - - internal class MultiClassificationCVRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public MultiClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is MultiClassMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var metrics = _context.MulticlassClassification.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch - { - MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, - MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, - MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, - MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, - MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - CrossValidationMetrics = metrics, - Metrics = res.Metrics, - }; - } - - throw new ArgumentException(); - } - } - - internal class RegressionTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public RegressionTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is RegressionMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.Regression.Evaluate(eval, metricSettings.LabelColumn, scoreColumnName: metricSettings.ScoreColumn); - - var metric = metricSettings.Metric switch - { - RegressionMetric.RootMeanSquaredError => metrics.RootMeanSquaredError, - RegressionMetric.RSquared => metrics.RSquared, - RegressionMetric.MeanSquaredError => metrics.MeanSquaredError, - RegressionMetric.MeanAbsoluteError => metrics.MeanAbsoluteError, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - } - - throw new ArgumentException(); - } - } - - internal class RegressionCVRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public RegressionCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is RegressionMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var metrics = _context.Regression.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch - { - RegressionMetric.RootMeanSquaredError => res.Metrics.RootMeanSquaredError, - RegressionMetric.RSquared => res.Metrics.RSquared, - RegressionMetric.MeanSquaredError => res.Metrics.MeanSquaredError, - RegressionMetric.MeanAbsoluteError => res.Metrics.MeanAbsoluteError, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - } - - throw new ArgumentException(); - } - } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index 0b8625f37e..f08e499d2f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -22,18 +22,21 @@ public class TrialResult } /// - /// TrialResult with Binary Classification Metrics + /// TrialResult with Metrics /// - internal class BinaryClassificationTrialResult : TrialResult + internal class TrialResult : TrialResult + where TMetric : class { - public BinaryClassificationMetrics BinaryClassificationMetrics { get; set; } + public TMetric Metrics { get; set; } - public IEnumerable> CrossValidationMetrics { get; set; } + public IEnumerable> CrossValidationMetrics { get; set; } public Exception Exception { get; set; } public bool IsSucceed { get => Exception == null; } public bool IsCrossValidation { get => CrossValidationMetrics == null; } + + public EstimatorChain Pipeline { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs index 7896ea5e9e..38d14c2a6d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs @@ -10,10 +10,6 @@ public class TrialSettings { public int TrialId { get; set; } - public SweepableEstimatorPipeline Pipeline { get; set; } - public Parameter Parameter { get; set; } - - public AutoMLExperiment.AutoMLExperimentSettings ExperimentSettings { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index 0231bfe978..1a1ecfaba3 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -96,26 +96,22 @@ public static int GetIndexOfBestScore(IEnumerable scores, bool isMetricM return isMetricMaximizing ? GetIndexOfMaxScore(scores) : GetIndexOfMinScore(scores); } - public static RunDetail ToRunDetail(MLContext context, TrialResult result) + public static RunDetail ToRunDetail(MLContext context, TrialResult result, SweepablePipeline pipeline) where TMetrics : class { - var pipeline = result.TrialSettings.Pipeline; - var trainerName = pipeline.ToString(); var parameter = result.TrialSettings.Parameter; - var estimator = pipeline.BuildTrainingPipeline(context, parameter); + var trainerName = pipeline.ToString(parameter); var modelContainer = new ModelContainer(context, result.Model); - return new RunDetail(trainerName, estimator, null, modelContainer, result.Metrics, result.Exception); + return new RunDetail(trainerName, result.Pipeline, null, modelContainer, result.Metrics, result.Exception); } - public static CrossValidationRunDetail ToCrossValidationRunDetail(MLContext context, TrialResult result) + public static CrossValidationRunDetail ToCrossValidationRunDetail(MLContext context, TrialResult result, SweepablePipeline pipeline) where TMetrics : class { - var pipeline = result.TrialSettings.Pipeline; - var trainerName = pipeline.ToString(); var parameter = result.TrialSettings.Parameter; - var estimator = pipeline.BuildTrainingPipeline(context, parameter); + var trainerName = pipeline.ToString(parameter); var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult(new ModelContainer(context, m.Model), m.Metrics, result.Exception)); - return new CrossValidationRunDetail(trainerName, estimator, null, crossValidationResult); + return new CrossValidationRunDetail(trainerName, result.Pipeline, null, crossValidationResult); } private static int GetIndexOfMinScore(IEnumerable scores) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 0e1c2de6b2..6e3fb4a682 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading; using FluentAssertions; -using FluentAssertions; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; @@ -228,7 +227,7 @@ public void AutoFitMultiClassification_Image() .CreateMulticlassClassificationExperiment(100) .Execute(trainData, columnInference.ColumnInformation); - Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.3, 0.9); + Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.1, 0.9); var scoredData = result.BestRun.Model.Transform(trainData); Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index ded7c0b7ae..fd927803c5 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -29,7 +29,7 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a var context = new MLContext(1); var pipeline = context.Transforms.Concatenate("Features", "Features") .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(context, 5); + var dummyTrainer = new DummyTrialRunner(5); var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) @@ -59,7 +59,7 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled var pipeline = context.Transforms.Concatenate("Features", "Features") .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(context, 1); + var dummyTrainer = new DummyTrialRunner(1); var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) @@ -88,7 +88,7 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() var pipeline = context.Transforms.Concatenate("Features", "Features") .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(context, 1); + var dummyTrainer = new DummyTrialRunner(1); var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) @@ -276,18 +276,15 @@ private IDataView GetDummyData() class DummyTrialRunner : ITrialRunner { private readonly int _finishAfterNSeconds; - private readonly MLContext _context; - public DummyTrialRunner(MLContext context, int finishAfterNSeconds) + public DummyTrialRunner(int finishAfterNSeconds) { _finishAfterNSeconds = finishAfterNSeconds; - _context = context; } public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) { Task.Delay(_finishAfterNSeconds * 1000).Wait(); - settings.ExperimentSettings.CancellationToken.ThrowIfCancellationRequested(); return new TrialResult { From 6f4c95b2ca7bf57ac898fd33382974960f8e6fd4 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 16:06:19 -0700 Subject: [PATCH 09/25] disable lgbm trainer in test --- .../API/BinaryClassificationExperiment.cs | 10 ++- .../AutoMLExperiment/AutoMLExperiment.cs | 8 +- .../{TrialRunner.cs => ITrialRunner.cs} | 0 ...sts.AutoFeaturizer_image_test.received.txt | 51 ----------- ...ests.AutoFeaturizer_iris_test.received.txt | 35 -------- ...AutoFeaturizer_uci_adult_test.received.txt | 84 ------------------- ...Tests.ImagePathFeaturizerTest.received.txt | 42 ---------- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 61 ++++++++++++-- .../AutoMLExperimentTests.cs | 63 +++++++++----- 9 files changed, 108 insertions(+), 246 deletions(-) rename src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/{TrialRunner.cs => ITrialRunner.cs} (100%) delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 8854ab53fb..8d67e0d992 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -288,15 +288,21 @@ private protected override CrossValidationRunDetail private SweepablePipeline CreateBinaryClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { + var useSdca = this.Settings.Trainers.Contains(BinaryClassificationTrainer.SdcaLogisticRegression); + var uselbfgs = this.Settings.Trainers.Contains(BinaryClassificationTrainer.LbfgsLogisticRegression); + var useLgbm = this.Settings.Trainers.Contains(BinaryClassificationTrainer.LightGbm); + var useFastForest = this.Settings.Trainers.Contains(BinaryClassificationTrainer.FastForest); + var useFastTree = this.Settings.Trainers.Contains(BinaryClassificationTrainer.FastTree); + if (preFeaturizer != null) { return preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) - .Append(Context.Auto().BinaryClassification(columnInformation.LabelColumnName, Features)); + .Append(Context.Auto().BinaryClassification(labelColumnName: columnInformation.LabelColumnName, useSdca: useSdca, useFastTree: useFastTree, useLgbm: useLgbm, useLbfgs: uselbfgs, useFastForest: useFastForest, featureColumnName: Features)); } else { return Context.Auto().Featurizer(trainData, columnInformation, Features) - .Append(Context.Auto().BinaryClassification(columnInformation.LabelColumnName, Features)); + .Append(Context.Auto().BinaryClassification(labelColumnName: columnInformation.LabelColumnName, useSdca: useSdca, useFastTree: useFastTree, useLgbm: useLgbm, useLbfgs: uselbfgs, useFastForest: useFastForest, featureColumnName: Features)); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 9ff4e5a8f4..c7e861322a 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -47,6 +47,8 @@ private void Initialize() InitializeServiceCollection(); } + internal IServiceCollection ServiceCollection { get => _serviceCollection; } + public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) { _settings.MaxExperimentTimeInSeconds = trainingTimeInSeconds; @@ -117,7 +119,7 @@ public AutoMLExperiment SetMonitor() return this; } - internal AutoMLExperiment SetPipeline(SweepablePipeline pipeline) + public AutoMLExperiment SetPipeline(SweepablePipeline pipeline) { _settings.SearchSpace[PipelineSearchspaceName] = pipeline.SearchSpace; _serviceCollection.AddSingleton(pipeline); @@ -159,7 +161,7 @@ public AutoMLExperiment SetTrialRunner() return this; } - internal AutoMLExperiment SetTuner(TTuner proposer) + public AutoMLExperiment SetTuner(TTuner proposer) where TTuner : class, ITuner { var descriptor = new ServiceDescriptor(typeof(ITuner), proposer); @@ -176,7 +178,7 @@ internal AutoMLExperiment SetTuner(TTuner proposer) return this; } - internal AutoMLExperiment SetTuner() + public AutoMLExperiment SetTuner() where TTuner : class, ITuner { var descriptor = new ServiceDescriptor(typeof(ITuner), typeof(TTuner), ServiceLifetime.Singleton); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs similarity index 100% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/TrialRunner.cs rename to src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt deleted file mode 100644 index d9697352c9..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.received.txt +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema": "e0 * e1 * e2 * e3 * e4", - "currentSchema": "e0 * e1 * e2 * e3 * e4", - "estimators": { - "e0": { - "estimatorType": "LoadImages", - "parameter": { - "OutputColumnName": "ImagePath", - "InputColumnName": "ImagePath" - } - }, - "e1": { - "estimatorType": "ResizeImages", - "parameter": { - "OutputColumnName": "ImagePath", - "InputColumnName": "ImagePath", - "ImageHeight": 224, - "ImageWidth": 224, - "CropAnchor": "Center", - "Resizing": "Fill" - } - }, - "e2": { - "estimatorType": "ExtractPixels", - "parameter": { - "OutputColumnName": "ImagePath", - "InputColumnName": "ImagePath", - "ColorsToExtract": "Rgb", - "OrderOfExtraction": "ARGB", - "OutputAsFloatArray": true - } - }, - "e3": { - "estimatorType": "DnnFeaturizerImage", - "parameter": { - "OutputColumnName": "ImagePath", - "InputColumnName": "ImagePath", - "ModelFactory": "resnet_18" - } - }, - "e4": { - "estimatorType": "Concatenate", - "parameter": { - "InputColumnNames": [ - "ImagePath" - ], - "OutputColumnName": "Features" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt deleted file mode 100644 index 0ec6f552d5..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.received.txt +++ /dev/null @@ -1,35 +0,0 @@ -{ - "schema": "e0 * e1", - "currentSchema": "e0 * e1", - "estimators": { - "e0": { - "estimatorType": "ReplaceMissingValues", - "parameter": { - "OutputColumnNames": [ - "col1", - "col2", - "col3", - "col4" - ], - "InputColumnNames": [ - "col1", - "col2", - "col3", - "col4" - ] - } - }, - "e1": { - "estimatorType": "Concatenate", - "parameter": { - "InputColumnNames": [ - "col1", - "col2", - "col3", - "col4" - ], - "OutputColumnName": "Features" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt deleted file mode 100644 index 17c65fe6ad..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.received.txt +++ /dev/null @@ -1,84 +0,0 @@ -{ - "schema": "e0 * (e1 \u002B e2) * e3", - "currentSchema": "e0 * e1 * e3", - "estimators": { - "e0": { - "estimatorType": "ReplaceMissingValues", - "parameter": { - "OutputColumnNames": [ - "Features" - ], - "InputColumnNames": [ - "Features" - ] - } - }, - "e1": { - "estimatorType": "OneHotEncoding", - "parameter": { - "OutputColumnNames": [ - "Workclass", - "education", - "marital-status", - "occupation", - "relationship", - "ethnicity", - "sex", - "native-country-region" - ], - "InputColumnNames": [ - "Workclass", - "education", - "marital-status", - "occupation", - "relationship", - "ethnicity", - "sex", - "native-country-region" - ] - } - }, - "e2": { - "estimatorType": "OneHotHashEncoding", - "parameter": { - "OutputColumnNames": [ - "Workclass", - "education", - "marital-status", - "occupation", - "relationship", - "ethnicity", - "sex", - "native-country-region" - ], - "InputColumnNames": [ - "Workclass", - "education", - "marital-status", - "occupation", - "relationship", - "ethnicity", - "sex", - "native-country-region" - ] - } - }, - "e3": { - "estimatorType": "Concatenate", - "parameter": { - "InputColumnNames": [ - "Features", - "Workclass", - "education", - "marital-status", - "occupation", - "relationship", - "ethnicity", - "sex", - "native-country-region" - ], - "OutputColumnName": "OutputFeature" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt deleted file mode 100644 index bf06bdba19..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.received.txt +++ /dev/null @@ -1,42 +0,0 @@ -{ - "schema": "e0 * e1 * e2 * e3", - "currentSchema": "e0 * e1 * e2 * e3", - "estimators": { - "e0": { - "estimatorType": "LoadImages", - "parameter": { - "OutputColumnName": "imagePath", - "InputColumnName": "imagePath" - } - }, - "e1": { - "estimatorType": "ResizeImages", - "parameter": { - "OutputColumnName": "imagePath", - "InputColumnName": "imagePath", - "ImageHeight": 224, - "ImageWidth": 224, - "CropAnchor": "Center", - "Resizing": "Fill" - } - }, - "e2": { - "estimatorType": "ExtractPixels", - "parameter": { - "OutputColumnName": "imagePath", - "InputColumnName": "imagePath", - "ColorsToExtract": "Rgb", - "OrderOfExtraction": "ARGB", - "OutputAsFloatArray": true - } - }, - "e3": { - "estimatorType": "DnnFeaturizerImage", - "parameter": { - "OutputColumnName": "imagePath", - "InputColumnName": "imagePath", - "ModelFactory": "resnet_18" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 6e3fb4a682..be056bf69b 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -14,6 +14,7 @@ using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.TestFrameworkCommon.Attributes; +using Microsoft.VisualBasic; using Xunit; using Xunit.Abstractions; using static Microsoft.ML.DataOperationsCatalog; @@ -37,7 +38,7 @@ private void MlContextLog(object sender, LoggingEventArgs e) _markerAutoFitContextLogTest = true; } - [LightGBMFact] + [Fact] public void AutoFit_UCI_Adult_Test() { var context = new MLContext(1); @@ -45,8 +46,17 @@ public void AutoFit_UCI_Adult_Test() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + settings.Trainers.Remove(BinaryClassificationTrainer.SdcaLogisticRegression); + settings.Trainers.Remove(BinaryClassificationTrainer.LbfgsLogisticRegression); + var result = context.Auto() - .CreateBinaryClassificationExperiment(1) + .CreateBinaryClassificationExperiment(settings) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -54,7 +64,7 @@ public void AutoFit_UCI_Adult_Test() Assert.NotNull(result.BestRun.TrainerName); } - [LightGBMFact] + [Fact] public void AutoFit_UCI_Adult_Train_Test_Split_Test() { var context = new MLContext(1); @@ -63,8 +73,17 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); var dataTrainTest = context.Data.TrainTestSplit(trainData); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + settings.Trainers.Remove(BinaryClassificationTrainer.SdcaLogisticRegression); + settings.Trainers.Remove(BinaryClassificationTrainer.LbfgsLogisticRegression); + var result = context.Auto() - .CreateBinaryClassificationExperiment(1) + .CreateBinaryClassificationExperiment(settings) .Execute(dataTrainTest.TrainSet, dataTrainTest.TestSet, DatasetUtil.UciAdultLabel); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -72,7 +91,7 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() Assert.NotNull(result.BestRun.TrainerName); } - [LightGBMFact] + [Fact] public void AutoFit_UCI_Adult_CrossValidation_10_Test() { var context = new MLContext(1); @@ -80,8 +99,16 @@ public void AutoFit_UCI_Adult_CrossValidation_10_Test() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + settings.Trainers.Remove(BinaryClassificationTrainer.SdcaLogisticRegression); + settings.Trainers.Remove(BinaryClassificationTrainer.LbfgsLogisticRegression); var result = context.Auto() - .CreateBinaryClassificationExperiment(1) + .CreateBinaryClassificationExperiment(settings) .Execute(trainData, 10, DatasetUtil.UciAdultLabel); Assert.True(result.BestRun.Results.Select(x => x.ValidationMetrics.Accuracy).Min() > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -138,8 +165,17 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) // When setting numberOfCVFolds // The results object is a CrossValidationExperimentResults<> object uint numberOfCVFolds = 5; + var settings = new MulticlassExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm); + settings.Trainers.Remove(MulticlassClassificationTrainer.SdcaMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); var result = context.Auto() - .CreateMulticlassClassificationExperiment(5) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainData, numberOfCVFolds, DatasetUtil.TrivialMulticlassDatasetLabel); result.BestRun.Results.First().ValidationMetrics.MicroAccuracy.Should().BeGreaterThan(0.7); @@ -157,8 +193,17 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) int crossValRowCountThreshold = 15000; trainData = context.Data.TakeRows(trainData, crossValRowCountThreshold - 1); + var settings = new MulticlassExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm); + settings.Trainers.Remove(MulticlassClassificationTrainer.SdcaMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); var result = context.Auto() - .CreateMulticlassClassificationExperiment(10) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainData, DatasetUtil.TrivialMulticlassDatasetLabel); Assert.True(result.BestRun.ValidationMetrics.MicroAccuracy >= 0.7); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index fd927803c5..665e70da16 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using FluentAssertions; using Microsoft.Data.Analysis; +using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Xunit; @@ -29,13 +30,12 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a var context = new MLContext(1); var pipeline = context.Transforms.Concatenate("Features", "Features") .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(5); var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") .SetTrainingTimeInSeconds(1) - .SetTrialRunner(dummyTrainer); + .UseDummyTrialRunner(5); var cts = new CancellationTokenSource(); @@ -59,13 +59,12 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled var pipeline = context.Transforms.Concatenate("Features", "Features") .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(1); var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") .SetTrainingTimeInSeconds(100) - .SetTrialRunner(dummyTrainer); + .UseDummyTrialRunner(1); var cts = new CancellationTokenSource(); @@ -88,13 +87,12 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() var pipeline = context.Transforms.Concatenate("Features", "Features") .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(1); var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") .SetTrainingTimeInSeconds(5) - .SetTrialRunner(dummyTrainer); + .UseDummyTrialRunner(1); var cts = new CancellationTokenSource(); cts.CancelAfter(10 * 1000); @@ -272,27 +270,50 @@ private IDataView GetDummyData() return df; } + } - class DummyTrialRunner : ITrialRunner + static class AutoMLExperimentExtension + { + public static AutoMLExperiment UseDummyTrialRunner(this AutoMLExperiment experiment, int finishAfterNSeconds) { - private readonly int _finishAfterNSeconds; - - public DummyTrialRunner(int finishAfterNSeconds) + var settings = new DummyTrialRunner.DummyTrialRunnerSettings { - _finishAfterNSeconds = finishAfterNSeconds; - } + FinishAfterNSeconds = finishAfterNSeconds, + }; + + experiment.ServiceCollection.AddSingleton(settings); + experiment.SetTrialRunner(); + + return experiment; + } + } + + class DummyTrialRunner : ITrialRunner + { + private readonly int _finishAfterNSeconds; + private readonly CancellationToken _ct; - public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) + public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, DummyTrialRunnerSettings settings) + { + _finishAfterNSeconds = settings.FinishAfterNSeconds; + _ct = automlSettings.CancellationToken; + } + + public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) + { + Task.Delay(_finishAfterNSeconds * 1000).Wait(_ct); + _ct.ThrowIfCancellationRequested(); + return new TrialResult { - Task.Delay(_finishAfterNSeconds * 1000).Wait(); + TrialSettings = settings, + DurationInMilliseconds = _finishAfterNSeconds * 1000, + Metric = 1.000 + 0.01 * settings.TrialId, + }; + } - return new TrialResult - { - TrialSettings = settings, - DurationInMilliseconds = _finishAfterNSeconds * 1000, - Metric = 1.000 + 0.01 * settings.TrialId, - }; - } + public class DummyTrialRunnerSettings + { + public int FinishAfterNSeconds { get; set; } } } } From 2eab5cbcdb7caf050d494a42888473fab139e93e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 16:39:31 -0700 Subject: [PATCH 10/25] clean up --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 2 +- .../API/BinaryClassificationExperiment.cs | 16 ++++++------- .../API/MulticlassClassificationExperiment.cs | 24 +++++++++---------- .../SweepableEstimator/SweepablePipeline.cs | 5 ---- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 15 ++++++++---- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index 510a1761cf..2abfedfe04 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -704,7 +704,7 @@ public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "F /// input data. /// column information. /// output feature column. - /// A for featurization. + /// A for featurization. public SweepablePipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features") { Contracts.CheckValue(data, nameof(data)); diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 8d67e0d992..49037f8f0f 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -222,7 +222,7 @@ public override ExperimentResult Execute(IDataView LabelColumnName = labelColumnName, }; - return this.Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); } public override ExperimentResult Execute(IDataView trainData, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -233,7 +233,7 @@ public override ExperimentResult Execute(IDataView SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, columnInformation, preFeaturizer, progressHandler); } public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -273,7 +273,7 @@ public override CrossValidationExperimentResult Exe SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); } private protected override RunDetail GetBestRun(IEnumerable> results) @@ -288,11 +288,11 @@ private protected override CrossValidationRunDetail private SweepablePipeline CreateBinaryClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { - var useSdca = this.Settings.Trainers.Contains(BinaryClassificationTrainer.SdcaLogisticRegression); - var uselbfgs = this.Settings.Trainers.Contains(BinaryClassificationTrainer.LbfgsLogisticRegression); - var useLgbm = this.Settings.Trainers.Contains(BinaryClassificationTrainer.LightGbm); - var useFastForest = this.Settings.Trainers.Contains(BinaryClassificationTrainer.FastForest); - var useFastTree = this.Settings.Trainers.Contains(BinaryClassificationTrainer.FastTree); + var useSdca = Settings.Trainers.Contains(BinaryClassificationTrainer.SdcaLogisticRegression); + var uselbfgs = Settings.Trainers.Contains(BinaryClassificationTrainer.LbfgsLogisticRegression); + var useLgbm = Settings.Trainers.Contains(BinaryClassificationTrainer.LightGbm); + var useFastForest = Settings.Trainers.Contains(BinaryClassificationTrainer.FastForest); + var useFastTree = Settings.Trainers.Contains(BinaryClassificationTrainer.FastTree); if (preFeaturizer != null) { diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 2fb3b8e990..36ea112247 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -156,10 +156,10 @@ public override ExperimentResult Execute(IDataV else { var splitData = Context.Data.TrainTestSplit(trainData); - return this.Execute(splitData.TrainSet, splitData.TestSet, columnInformation, preFeaturizer, progressHandler); + return Execute(splitData.TrainSet, splitData.TestSet, columnInformation, preFeaturizer, progressHandler); } - _pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); var monitor = new TrialResultMonitor(Context, _pipeline); @@ -187,7 +187,7 @@ public override ExperimentResult Execute(IDataV _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); - _pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); var monitor = new TrialResultMonitor(Context, _pipeline); monitor.OnTrialCompleted += (o, e) => @@ -214,7 +214,7 @@ public override ExperimentResult Execute(IDataV LabelColumnName = labelColumnName, }; - return this.Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); } public override ExperimentResult Execute(IDataView trainData, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -225,7 +225,7 @@ public override ExperimentResult Execute(IDataV SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, columnInformation, preFeaturizer, progressHandler); } public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -235,7 +235,7 @@ public override CrossValidationExperimentResult _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); - _pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); var monitor = new TrialResultMonitor(Context, _pipeline); @@ -266,7 +266,7 @@ public override CrossValidationExperimentResult SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); } private protected override CrossValidationRunDetail GetBestCrossValRun(IEnumerable> results) @@ -281,11 +281,11 @@ private protected override RunDetail GetBestRun private SweepablePipeline CreateMulticlassClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { - var useSdca = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.SdcaMaximumEntropy); - var uselbfgs = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); - var useLgbm = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.LightGbm); - var useFastForest = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.FastForestOva); - var useFastTree = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.FastTreeOva); + var useSdca = Settings.Trainers.Contains(MulticlassClassificationTrainer.SdcaMaximumEntropy); + var uselbfgs = Settings.Trainers.Contains(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); + var useLgbm = Settings.Trainers.Contains(MulticlassClassificationTrainer.LightGbm); + var useFastForest = Settings.Trainers.Contains(MulticlassClassificationTrainer.FastForestOva); + var useFastTree = Settings.Trainers.Contains(MulticlassClassificationTrainer.FastTreeOva); SweepablePipeline pipeline = new SweepablePipeline(); if (preFeaturizer != null) diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs index d70c48868c..6f754d12b5 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs @@ -69,11 +69,6 @@ internal SweepablePipeline() _schema = null; } - internal SweepablePipeline(MultiModelPipeline pipeline) - : this(pipeline.Estimators, pipeline.Schema) - { - } - internal SweepablePipeline(Dictionary estimators, Entity schema, string currentSchema = null) { _estimators = estimators; diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index be056bf69b..42257323e2 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -510,9 +510,17 @@ public void AutoFitWithPresplittedData() var dataFull = textLoader.Load(dataPath); var dataTrainTest = context.Data.TrainTestSplit(dataFull); var dataCV = context.Data.CrossValidationSplit(dataFull, numberOfFolds: 2); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 10, + }; + + // remove fastForest because it doesn't calibrate score + // so column "probability" will be missing in the final result; + settings.Trainers.Remove(BinaryClassificationTrainer.FastForest); var modelFull = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(settings) .Execute(dataFull, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun @@ -522,7 +530,7 @@ public void AutoFitWithPresplittedData() // Therefore, we need to create a new context after an experiment is completed. context = new MLContext(1); var modelTrainTest = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(settings) .Execute(dataTrainTest.TrainSet, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun @@ -530,7 +538,7 @@ public void AutoFitWithPresplittedData() context = new MLContext(1); var modelCV = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(settings) .Execute(dataCV.First().TrainSet, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun @@ -543,7 +551,6 @@ public void AutoFitWithPresplittedData() var resFull = model.Transform(dataFull); var resTrainTest = model.Transform(dataTrainTest.TrainSet); var resCV = model.Transform(dataCV.First().TrainSet); - Assert.Equal(31, resFull.Schema.Count); Assert.Equal(31, resTrainTest.Schema.Count); Assert.Equal(31, resCV.Schema.Count); From 928f98f4f7fadf827df13a63ad44f1baefa79e7a Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 17 Aug 2022 17:08:40 -0700 Subject: [PATCH 11/25] renmame SetEvaluateMetricManager --- .../Microsoft.ML.AutoML.Samples/Cifar10.cs | 2 +- .../API/BinaryClassificationExperiment.cs | 6 ++--- .../API/MulticlassClassificationExperiment.cs | 6 ++--- .../AutoMLExperiment/AutoMLExperiment.cs | 23 ++++++++++++------- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- .../AutoMLExperimentTests.cs | 18 +++++++-------- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs index 2f4254ab8a..cda207ec09 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs @@ -40,7 +40,7 @@ public static void Run() experiment.SetDataset(trainDataset, testDataset) .SetPipeline(pipeline) - .SetEvaluateMetric(MulticlassClassificationMetric.MicroAccuracy) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MicroAccuracy) .SetTrainingTimeInSeconds(200); var result = experiment.Run(); diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 49037f8f0f..8eb87a9b41 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -148,7 +148,7 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); // Cross val threshold for # of dataset rows -- @@ -192,7 +192,7 @@ public override ExperimentResult Execute(IDataView public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); @@ -239,7 +239,7 @@ public override ExperimentResult Execute(IDataView public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 36ea112247..28ed33071a 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -138,7 +138,7 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); // Cross val threshold for # of dataset rows -- @@ -183,7 +183,7 @@ public override ExperimentResult Execute(IDataV public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); @@ -231,7 +231,7 @@ public override ExperimentResult Execute(IDataV public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index c7e861322a..388f2cf30b 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -39,7 +39,6 @@ private void InitializeServiceCollection() _serviceCollection.TryAddSingleton(_context); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddSingleton(); } private void Initialize() @@ -207,16 +206,16 @@ public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) return this; } - public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + public AutoMLExperiment SetBinaryClassificationMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { var metricManager = new BinaryMetricManager(metric, predictedColumn, labelColumn); - _serviceCollection.AddSingleton(metricManager); - _serviceCollection.AddSingleton(metricManager); + this.SetEvaluateMetric(metricManager); + return this; } - public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + public AutoMLExperiment SetMulticlassClassificationMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { var metricManager = new MultiClassMetricManager() { @@ -224,13 +223,13 @@ public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, PredictedColumn = predictedColumn, LabelColumn = labelColumn, }; - _serviceCollection.AddSingleton(metricManager); - _serviceCollection.AddSingleton(metricManager); + this.SetEvaluateMetric(metricManager); + return this; } - public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") + public AutoMLExperiment SetRegressionMetric(RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") { var metricManager = new RegressionMetricManager() { @@ -238,6 +237,14 @@ public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelC ScoreColumn = scoreColumn, LabelColumn = labelColumn, }; + this.SetEvaluateMetric(metricManager); + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(TEvaluateMetricManager metricManager) + where TEvaluateMetricManager : class, IEvaluateMetricManager + { _serviceCollection.AddSingleton(metricManager); _serviceCollection.AddSingleton(metricManager); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 42257323e2..9ba507e555 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -136,7 +136,7 @@ public void SweepablePipeline_AutoFit_UCI_Adult_CrossValidation_10_Test() experiment.SetPipeline(pipeline) .SetDataset(trainData, 5) - .SetEvaluateMetric(BinaryClassificationMetric.Accuracy, DatasetUtil.UciAdultLabel) + .SetBinaryClassificationMetric(BinaryClassificationMetric.Accuracy, DatasetUtil.UciAdultLabel) .SetTrainingTimeInSeconds(10); var res = experiment.Run(); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 665e70da16..52f83eb22e 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -33,7 +33,7 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") + .SetRegressionMetric(RegressionMetric.RootMeanSquaredError, "Label") .SetTrainingTimeInSeconds(1) .UseDummyTrialRunner(5); @@ -62,7 +62,7 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") + .SetRegressionMetric(RegressionMetric.RootMeanSquaredError, "Label") .SetTrainingTimeInSeconds(100) .UseDummyTrialRunner(1); @@ -90,7 +90,7 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() var experiment = context.Auto().CreateExperiment(); experiment.SetPipeline(pipeline) .SetDataset(GetDummyData(), 10) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") + .SetRegressionMetric(RegressionMetric.RootMeanSquaredError, "Label") .SetTrainingTimeInSeconds(5) .UseDummyTrialRunner(1); @@ -119,7 +119,7 @@ public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(context.Data.TrainTestSplit(data)) - .SetEvaluateMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) .SetTrainingTimeInSeconds(10); @@ -144,7 +144,7 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(data, 5) - .SetEvaluateMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) .SetTrainingTimeInSeconds(10); @@ -171,7 +171,7 @@ public async Task AutoMLExperiment_Iris_CV_5_Test() .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(data, 5) - .SetEvaluateMetric(MulticlassClassificationMetric.MacroAccuracy, label) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, label) .SetPipeline(pipeline) .SetTrainingTimeInSeconds(10); @@ -198,7 +198,7 @@ public async Task AutoMLExperiment_Iris_Train_Test_Split_Test() .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(context.Data.TrainTestSplit(data)) - .SetEvaluateMetric(MulticlassClassificationMetric.MacroAccuracy, label) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, label) .SetPipeline(pipeline) .SetTrainingTimeInSeconds(10); @@ -225,7 +225,7 @@ public async Task AutoMLExperiment_Taxi_Fare_Train_Test_Split_Test() .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(train, test) - .SetEvaluateMetric(RegressionMetric.RSquared, label) + .SetRegressionMetric(RegressionMetric.RSquared, label) .SetPipeline(pipeline) .SetTrainingTimeInSeconds(50); @@ -251,7 +251,7 @@ public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test() .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgs: false)); experiment.SetDataset(train, 5) - .SetEvaluateMetric(RegressionMetric.RSquared, label) + .SetRegressionMetric(RegressionMetric.RSquared, label) .SetPipeline(pipeline) .SetTrainingTimeInSeconds(50); From c1bc1b951dad2a1a8cdd12aca28b70c15df66545 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 18 Aug 2022 17:14:34 -0700 Subject: [PATCH 12/25] only keep back-to-bone function in AutoMLExperiment, and make other API as extension --- .../Microsoft.ML.AutoML.Samples/Cifar10.cs | 1 + .../API/AutoMLExperimentExtension.cs | 113 +++++++++++ .../API/BinaryClassificationExperiment.cs | 52 +++-- .../API/MulticlassClassificationExperiment.cs | 43 ++++- .../AutoMLExperiment/AutoMLExperiment.cs | 179 +++++++----------- .../AutoMLExperiment/IMonitor.cs | 10 +- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 5 +- .../AutoMLExperimentTests.cs | 87 +++------ 8 files changed, 285 insertions(+), 205 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs index cda207ec09..d82fb2b44b 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using Microsoft.ML.AutoML.API; namespace Microsoft.ML.AutoML.Samples { diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs new file mode 100644 index 0000000000..5b43285b70 --- /dev/null +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using static Microsoft.ML.DataOperationsCatalog; + +namespace Microsoft.ML.AutoML.API +{ + public static class AutoMLExperimentExtension + { + + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView test) + { + var datasetManager = new TrainTestDatasetManager() + { + TrainDataset = train, + TestDataset = test + }; + + experiment.ServiceCollection.AddSingleton(datasetManager); + experiment.ServiceCollection.AddSingleton(datasetManager); + + return experiment; + } + + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, TrainTestData trainTestSplit) + { + return experiment.SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet); + } + + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView dataset, int fold = 10) + { + var datasetManager = new CrossValidateDatasetManager() + { + Dataset = dataset, + Fold = fold, + }; + + experiment.ServiceCollection.AddSingleton(datasetManager); + experiment.ServiceCollection.AddSingleton(datasetManager); + + return experiment; + } + + + public static AutoMLExperiment SetBinaryClassificationMetric(this AutoMLExperiment experiment, BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + { + var metricManager = new BinaryMetricManager(metric, predictedColumn, labelColumn); + return experiment.SetEvaluateMetric(metricManager); + } + + public static AutoMLExperiment SetMulticlassClassificationMetric(this AutoMLExperiment experiment, MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + { + var metricManager = new MultiClassMetricManager() + { + Metric = metric, + PredictedColumn = predictedColumn, + LabelColumn = labelColumn, + }; + + return experiment.SetEvaluateMetric(metricManager); + } + + public static AutoMLExperiment SetRegressionMetric(this AutoMLExperiment experiment, RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") + { + var metricManager = new RegressionMetricManager() + { + Metric = metric, + ScoreColumn = scoreColumn, + LabelColumn = labelColumn, + }; + + return experiment.SetEvaluateMetric(metricManager); + } + + public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, SweepablePipeline pipeline) + { + experiment.AddSearchSpace(AutoMLExperiment.PipelineSearchspaceName, pipeline.SearchSpace); + experiment.ServiceCollection.AddSingleton(pipeline); + + experiment.SetTrialRunner(); + experiment.SetMonitor(); + experiment.SetTuner(); + + return experiment; + } + + public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, SweepableEstimatorPipeline pipeline) + { + var res = new SweepablePipeline(); + foreach (var e in pipeline.Estimators) + { + res = res.Append(e); + } + + return experiment.SetPipeline(res); + } + + private static AutoMLExperiment SetEvaluateMetric(this AutoMLExperiment experiment, TEvaluateMetricManager metricManager) + where TEvaluateMetricManager : class, IEvaluateMetricManager + { + experiment.ServiceCollection.AddSingleton(metricManager); + experiment.ServiceCollection.AddSingleton(metricManager); + experiment.SetIsMaximize(metricManager.IsMaximize); + + return experiment; + } + } +} diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 8eb87a9b41..84975aede2 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -7,6 +7,7 @@ using System.Diagnostics; using System.Linq; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; @@ -148,6 +149,7 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; + TrialResultMonitor monitor = null; _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); @@ -169,17 +171,18 @@ public override ExperimentResult Execute(IDataView _experiment.SetDataset(splitData.TrainSet, splitData.TestSet); } _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(_pipeline); - - var monitor = new TrialResultMonitor(Context, _pipeline); - monitor.OnTrialCompleted += (o, e) => + _experiment.SetMonitor((provider) => { - var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); - progressHandler?.Report(detail); - }; + monitor = provider.GetService>(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + return monitor; + }); _experiment.SetTrialRunner(); - _experiment.SetMonitor(monitor); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); @@ -192,19 +195,23 @@ public override ExperimentResult Execute(IDataView public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; + TrialResultMonitor monitor = null; _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context, _pipeline); - monitor.OnTrialCompleted += (o, e) => + _experiment.SetMonitor((provider) => { - var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); - progressHandler?.Report(detail); - }; + monitor = provider.GetService>(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; - _experiment.SetMonitor(monitor); + return monitor; + }); _experiment.SetTrialRunner(); _experiment.Run(); @@ -239,21 +246,24 @@ public override ExperimentResult Execute(IDataView public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; + TrialResultMonitor monitor = null; _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - - var monitor = new TrialResultMonitor(Context, _pipeline); - monitor.OnTrialCompleted += (o, e) => + _experiment.SetMonitor((provider) => { - var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); + monitor = provider.GetService>(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; - progressHandler?.Report(runDetails); - }; + return monitor; + }); - _experiment.SetMonitor(monitor); _experiment.SetTrialRunner(); _experiment.Run(); diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 28ed33071a..915bf88fde 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; @@ -138,6 +140,7 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; + TrialResultMonitor monitor = null; _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); @@ -161,8 +164,17 @@ public override ExperimentResult Execute(IDataV _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + _experiment.SetMonitor((provider) => + { + monitor = provider.GetService>(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; - var monitor = new TrialResultMonitor(Context, _pipeline); + return monitor; + }); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); @@ -183,13 +195,24 @@ public override ExperimentResult Execute(IDataV public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; + TrialResultMonitor monitor = null; _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context, _pipeline); + _experiment.SetMonitor((provider) => + { + monitor = provider.GetService>(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); @@ -231,20 +254,24 @@ public override ExperimentResult Execute(IDataV public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; + TrialResultMonitor monitor = null; _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); - - var monitor = new TrialResultMonitor(Context, _pipeline); - monitor.OnTrialCompleted += (o, e) => + _experiment.SetMonitor((provider) => { - var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); + monitor = provider.GetService>(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; - progressHandler?.Report(runDetails); - }; + return monitor; + }); _experiment.SetTrialRunner(); _experiment.SetMonitor(monitor); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 388f2cf30b..1913020381 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.SearchSpace; using static Microsoft.ML.DataOperationsCatalog; @@ -21,11 +22,18 @@ public class AutoMLExperiment private double _bestError = double.MaxValue; private TrialResult _bestTrialResult = null; private readonly IServiceCollection _serviceCollection; + private CancellationTokenSource _cts; public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { _context = context; _settings = settings; + + if (_settings.Seed == null) + { + _settings.Seed = ((IHostEnvironmentInternal)_context.Model.GetEnvironment()).Seed; + } + if (_settings.SearchSpace == null) { _settings.SearchSpace = new SearchSpace.SearchSpace(); @@ -36,13 +44,26 @@ public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) private void InitializeServiceCollection() { - _serviceCollection.TryAddSingleton(_context); + _serviceCollection.TryAddSingleton((provider) => + { + var context = new MLContext(_settings.Seed); + _cts.Token.Register(() => + { + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + context.CancelExecution(); + }); + + return context; + }); _serviceCollection.TryAddSingleton(_settings); - _serviceCollection.TryAddSingleton(); + _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } private void Initialize() { + _cts = new CancellationTokenSource(); InitializeServiceCollection(); } @@ -54,35 +75,15 @@ public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) return this; } - public AutoMLExperiment SetDataset(IDataView train, IDataView test) + public AutoMLExperiment SetIsMaximize(bool isMaximize) { - var datasetManager = new TrainTestDatasetManager() - { - TrainDataset = train, - TestDataset = test - }; - - _serviceCollection.AddSingleton(datasetManager); - + _settings.IsMaximize = isMaximize; return this; } - public AutoMLExperiment SetDataset(TrainTestData trainTestSplit) + public AutoMLExperiment AddSearchSpace(string key, SearchSpace.SearchSpace searchSpace) { - SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet); - - return this; - } - - public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) - { - var datasetManager = new CrossValidateDatasetManager() - { - Dataset = dataset, - Fold = fold, - }; - - _serviceCollection.AddSingleton(datasetManager); + _settings.SearchSpace[key] = searchSpace; return this; } @@ -118,13 +119,19 @@ public AutoMLExperiment SetMonitor() return this; } - public AutoMLExperiment SetPipeline(SweepablePipeline pipeline) + public AutoMLExperiment SetMonitor(Func factory) { - _settings.SearchSpace[PipelineSearchspaceName] = pipeline.SearchSpace; - _serviceCollection.AddSingleton(pipeline); + var descriptor = ServiceDescriptor.Singleton(typeof(IMonitor), factory); + + if (_serviceCollection.Contains(descriptor)) + { + _serviceCollection.Replace(descriptor); + } + else + { + _serviceCollection.Add(descriptor); + } - SetTrialRunner(); - SetMonitor(); return this; } @@ -144,10 +151,11 @@ public AutoMLExperiment SetTrialRunner(TTrialRunner runner) return this; } - public AutoMLExperiment SetTrialRunner() + public AutoMLExperiment SetTrialRunner(Func factory) where TTrialRunner : class, ITrialRunner { - var descriptor = new ServiceDescriptor(typeof(ITrialRunner), typeof(TTrialRunner), ServiceLifetime.Transient); + var descriptor = ServiceDescriptor.Singleton(typeof(ITrialRunner), factory); + if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -160,11 +168,10 @@ public AutoMLExperiment SetTrialRunner() return this; } - public AutoMLExperiment SetTuner(TTuner proposer) - where TTuner : class, ITuner + public AutoMLExperiment SetTrialRunner() + where TTrialRunner : class, ITrialRunner { - var descriptor = new ServiceDescriptor(typeof(ITuner), proposer); - + var descriptor = new ServiceDescriptor(typeof(ITrialRunner), typeof(TTrialRunner), ServiceLifetime.Transient); if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -177,10 +184,11 @@ public AutoMLExperiment SetTuner(TTuner proposer) return this; } - public AutoMLExperiment SetTuner() + public AutoMLExperiment SetTuner(TTuner proposer) where TTuner : class, ITuner { - var descriptor = new ServiceDescriptor(typeof(ITuner), typeof(TTuner), ServiceLifetime.Singleton); + var descriptor = new ServiceDescriptor(typeof(ITuner), proposer); + if (_serviceCollection.Contains(descriptor)) { _serviceCollection.Replace(descriptor); @@ -193,60 +201,18 @@ public AutoMLExperiment SetTuner() return this; } - public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) + public AutoMLExperiment SetTuner() + where TTuner : class, ITuner { - var res = new SweepablePipeline(); - foreach (var e in pipeline.Estimators) + var descriptor = new ServiceDescriptor(typeof(ITuner), typeof(TTuner), ServiceLifetime.Singleton); + if (_serviceCollection.Contains(descriptor)) { - res = res.Append(e); + _serviceCollection.Replace(descriptor); } - - SetPipeline(res); - - return this; - } - - public AutoMLExperiment SetBinaryClassificationMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") - { - var metricManager = new BinaryMetricManager(metric, predictedColumn, labelColumn); - this.SetEvaluateMetric(metricManager); - - - return this; - } - - public AutoMLExperiment SetMulticlassClassificationMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") - { - var metricManager = new MultiClassMetricManager() - { - Metric = metric, - PredictedColumn = predictedColumn, - LabelColumn = labelColumn, - }; - this.SetEvaluateMetric(metricManager); - - - return this; - } - - public AutoMLExperiment SetRegressionMetric(RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") - { - var metricManager = new RegressionMetricManager() + else { - Metric = metric, - ScoreColumn = scoreColumn, - LabelColumn = labelColumn, - }; - this.SetEvaluateMetric(metricManager); - - return this; - } - - public AutoMLExperiment SetEvaluateMetric(TEvaluateMetricManager metricManager) - where TEvaluateMetricManager : class, IEvaluateMetricManager - { - _serviceCollection.AddSingleton(metricManager); - _serviceCollection.AddSingleton(metricManager); + _serviceCollection.Add(descriptor); + } return this; } @@ -269,28 +235,18 @@ public TrialResult Run() public async Task RunAsync(CancellationToken ct = default) { ValidateSettings(); - var cts = new CancellationTokenSource(); - _settings.CancellationToken = ct; - cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); - _settings.CancellationToken.Register(() => cts.Cancel()); - cts.Token.Register(() => - { - // only force-canceling running trials when there's completed trials. - // otherwise, wait for the current running trial to be completed. - if (_bestTrialResult != null) - _context.CancelExecution(); - }); - Initialize(); + _settings.CancellationToken = ct; + _cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); + _settings.CancellationToken.Register(() => _cts.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); var trialNum = 0; var tuner = serviceProvider.GetService(); - var metricManager = serviceProvider.GetService(); - + Contracts.Assert(tuner != null, "tuner can't be null"); while (true) { - if (cts.Token.IsCancellationRequested) + if (_cts.Token.IsCancellationRequested) { break; } @@ -302,21 +258,22 @@ public async Task RunAsync(CancellationToken ct = default) var parameter = tuner.Propose(setting); setting.Parameter = parameter; - monitor.ReportRunningTrial(setting); + monitor?.ReportRunningTrial(setting); var runner = serviceProvider.GetService(); + Contracts.Assert(runner != null, "trial runner can't be null"); try { var trialResult = runner.Run(setting, serviceProvider); - monitor.ReportCompletedTrial(trialResult); + monitor?.ReportCompletedTrial(trialResult); tuner.Update(trialResult); - var error = metricManager.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) { _bestTrialResult = trialResult; _bestError = error; - monitor.ReportBestTrial(trialResult); + monitor?.ReportBestTrial(trialResult); } } catch (OperationCanceledException) @@ -327,7 +284,7 @@ public async Task RunAsync(CancellationToken ct = default) { monitor.ReportFailTrial(setting, ex); - if (!cts.IsCancellationRequested && _bestTrialResult == null) + if (!_cts.IsCancellationRequested && _bestTrialResult == null) { // TODO // it's questionable on whether to abort the entire training process @@ -362,7 +319,9 @@ public class AutoMLExperimentSettings : ExperimentSettings { public int? Seed { get; set; } - internal SearchSpace.SearchSpace SearchSpace { get; set; } + public SearchSpace.SearchSpace SearchSpace { get; set; } + + public bool IsMaximize { get; set; } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 2ba3efacd7..a39f7d6d39 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -25,14 +25,12 @@ public interface IMonitor // this monitor redirects output result to context.log internal class MLContextMonitor : IMonitor { - private readonly MLContext _context; private readonly IChannel _logger; private readonly List _completedTrials; private readonly SweepablePipeline _pipeline; - public MLContextMonitor(MLContext context, SweepablePipeline pipeline) + public MLContextMonitor(IChannel logger, SweepablePipeline pipeline) { - _context = context; - _logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); + _logger = logger; _completedTrials = new List(); _pipeline = pipeline; } @@ -62,8 +60,8 @@ public virtual void ReportRunningTrial(TrialSettings setting) internal class TrialResultMonitor : MLContextMonitor where TMetrics : class { - public TrialResultMonitor(MLContext context, SweepablePipeline pipeline) - : base(context, pipeline) + public TrialResultMonitor(IChannel channel, SweepablePipeline pipeline) + : base(channel, pipeline) { this.RunDetails = new List>(); } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 9ba507e555..8099059cad 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using FluentAssertions; +using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; @@ -526,9 +527,6 @@ public void AutoFitWithPresplittedData() .BestRun .Model; - // AutoMLExperiment can't run on canceled context. - // Therefore, we need to create a new context after an experiment is completed. - context = new MLContext(1); var modelTrainTest = context.Auto() .CreateBinaryClassificationExperiment(settings) .Execute(dataTrainTest.TrainSet, @@ -536,7 +534,6 @@ public void AutoFitWithPresplittedData() .BestRun .Model; - context = new MLContext(1); var modelCV = context.Auto() .CreateBinaryClassificationExperiment(settings) .Execute(dataCV.First().TrainSet, diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 52f83eb22e..7824ecf991 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -11,6 +11,7 @@ using FluentAssertions; using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.AutoML.API; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Xunit; @@ -28,14 +29,16 @@ public AutoMLExperimentTests(ITestOutputHelper output) : base(output) public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_and_no_trial_completed_Async() { var context = new MLContext(1); - var pipeline = context.Transforms.Concatenate("Features", "Features") - .Append(context.Auto().Regression()); var experiment = context.Auto().CreateExperiment(); - experiment.SetPipeline(pipeline) - .SetDataset(GetDummyData(), 10) - .SetRegressionMetric(RegressionMetric.RootMeanSquaredError, "Label") - .SetTrainingTimeInSeconds(1) - .UseDummyTrialRunner(5); + + experiment.SetTrainingTimeInSeconds(1) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 5, channel); + }) + .SetTuner(); var cts = new CancellationTokenSource(); @@ -56,15 +59,16 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async() { var context = new MLContext(1); - var pipeline = context.Transforms.Concatenate("Features", "Features") - .Append(context.Auto().Regression()); var experiment = context.Auto().CreateExperiment(); - experiment.SetPipeline(pipeline) - .SetDataset(GetDummyData(), 10) - .SetRegressionMetric(RegressionMetric.RootMeanSquaredError, "Label") - .SetTrainingTimeInSeconds(100) - .UseDummyTrialRunner(1); + experiment.SetTrainingTimeInSeconds(100) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 1, channel); + }) + .SetTuner(); var cts = new CancellationTokenSource(); @@ -84,15 +88,16 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() { var context = new MLContext(1); - var pipeline = context.Transforms.Concatenate("Features", "Features") - .Append(context.Auto().Regression()); var experiment = context.Auto().CreateExperiment(); - experiment.SetPipeline(pipeline) - .SetDataset(GetDummyData(), 10) - .SetRegressionMetric(RegressionMetric.RootMeanSquaredError, "Label") - .SetTrainingTimeInSeconds(5) - .UseDummyTrialRunner(1); + experiment.SetTrainingTimeInSeconds(5) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 1, channel); + }) + .SetTuner(); var cts = new CancellationTokenSource(); cts.CancelAfter(10 * 1000); @@ -258,49 +263,24 @@ public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test() var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.5); } - - private IDataView GetDummyData() - { - var x = Enumerable.Range(-10000, 10000).Select(value => value * 1f).ToArray(); - var y = x.Select(value => value * value); - - var df = new DataFrame(); - df["Features"] = DataFrameColumn.Create("Features", x); - df["Label"] = DataFrameColumn.Create("Label", y); - - return df; - } - } - - static class AutoMLExperimentExtension - { - public static AutoMLExperiment UseDummyTrialRunner(this AutoMLExperiment experiment, int finishAfterNSeconds) - { - var settings = new DummyTrialRunner.DummyTrialRunnerSettings - { - FinishAfterNSeconds = finishAfterNSeconds, - }; - - experiment.ServiceCollection.AddSingleton(settings); - experiment.SetTrialRunner(); - - return experiment; - } } class DummyTrialRunner : ITrialRunner { private readonly int _finishAfterNSeconds; private readonly CancellationToken _ct; + private readonly IChannel _logger; - public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, DummyTrialRunnerSettings settings) + public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) { - _finishAfterNSeconds = settings.FinishAfterNSeconds; + _finishAfterNSeconds = finishAfterNSeconds; _ct = automlSettings.CancellationToken; + _logger = logger; } public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) { + _logger.Info("Update Running Trial"); Task.Delay(_finishAfterNSeconds * 1000).Wait(_ct); _ct.ThrowIfCancellationRequested(); return new TrialResult @@ -310,10 +290,5 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) Metric = 1.000 + 0.01 * settings.TrialId, }; } - - public class DummyTrialRunnerSettings - { - public int FinishAfterNSeconds { get; set; } - } } } From 31362ff09ec2200195ee7d1a091df296802467d1 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 18 Aug 2022 17:35:57 -0700 Subject: [PATCH 13/25] fix di --- .../API/BinaryClassificationExperiment.cs | 13 ++++++++++--- .../API/MulticlassClassificationExperiment.cs | 13 ++++++++++--- .../AutoMLExperiment/AutoMLExperiment.cs | 5 +++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 84975aede2..fa91ee864c 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; +using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; @@ -173,7 +174,9 @@ public override ExperimentResult Execute(IDataView _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetMonitor((provider) => { - monitor = provider.GetService>(); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); @@ -203,7 +206,9 @@ public override ExperimentResult Execute(IDataView _experiment.SetPipeline(_pipeline); _experiment.SetMonitor((provider) => { - monitor = provider.GetService>(); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); @@ -254,7 +259,9 @@ public override CrossValidationExperimentResult Exe _experiment.SetPipeline(_pipeline); _experiment.SetMonitor((provider) => { - monitor = provider.GetService>(); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 915bf88fde..537dbe903d 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; +using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; @@ -166,7 +167,9 @@ public override ExperimentResult Execute(IDataV _experiment.SetPipeline(_pipeline); _experiment.SetMonitor((provider) => { - monitor = provider.GetService>(); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); @@ -204,7 +207,9 @@ public override ExperimentResult Execute(IDataV _experiment.SetPipeline(_pipeline); _experiment.SetMonitor((provider) => { - monitor = provider.GetService>(); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); @@ -263,7 +268,9 @@ public override CrossValidationExperimentResult _experiment.SetPipeline(_pipeline); _experiment.SetMonitor((provider) => { - monitor = provider.GetService>(); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); monitor.OnTrialCompleted += (o, e) => { var detail = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 1913020381..640475788e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -120,8 +120,9 @@ public AutoMLExperiment SetMonitor() } public AutoMLExperiment SetMonitor(Func factory) + where TMonitor : class, IMonitor { - var descriptor = ServiceDescriptor.Singleton(typeof(IMonitor), factory); + var descriptor = ServiceDescriptor.Singleton(factory); if (_serviceCollection.Contains(descriptor)) { @@ -154,7 +155,7 @@ public AutoMLExperiment SetTrialRunner(TTrialRunner runner) public AutoMLExperiment SetTrialRunner(Func factory) where TTrialRunner : class, ITrialRunner { - var descriptor = ServiceDescriptor.Singleton(typeof(ITrialRunner), factory); + var descriptor = ServiceDescriptor.Singleton(factory); if (_serviceCollection.Contains(descriptor)) { From 4c65fa355ba3588b65204bb5e5a9b9e54d3058d1 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 18 Aug 2022 17:39:03 -0700 Subject: [PATCH 14/25] clean up --- .../API/BinaryClassificationExperiment.cs | 1 + .../API/MulticlassClassificationExperiment.cs | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index fa91ee864c..97c656a72d 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -172,6 +172,7 @@ public override ExperimentResult Execute(IDataView _experiment.SetDataset(splitData.TrainSet, splitData.TestSet); } _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); _experiment.SetMonitor((provider) => { var channel = provider.GetService(); diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 537dbe903d..5ca2baa48a 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -185,7 +185,6 @@ public override ExperimentResult Execute(IDataV }; _experiment.SetTrialRunner(); - _experiment.SetMonitor(monitor); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); @@ -218,14 +217,8 @@ public override ExperimentResult Execute(IDataV return monitor; }); - monitor.OnTrialCompleted += (o, e) => - { - var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); - progressHandler?.Report(detail); - }; _experiment.SetTrialRunner(); - _experiment.SetMonitor(monitor); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); @@ -281,7 +274,6 @@ public override CrossValidationExperimentResult }); _experiment.SetTrialRunner(); - _experiment.SetMonitor(monitor); _experiment.Run(); var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline)); From c2368309919779ef2dfd5b96c8ca3fb7b44c9e3e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sat, 20 Aug 2022 22:41:41 -0700 Subject: [PATCH 15/25] convert vector to vector c.Purpose == ColumnPurpose.TextFeature); - var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature && data.Schema[c.ColumnIndex].Type != BooleanDataViewType.Instance); - var booleanFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature && data.Schema[c.ColumnIndex].Type == BooleanDataViewType.Instance); + var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature + && data.Schema[c.ColumnIndex].Type != BooleanDataViewType.Instance + && !(data.Schema[c.ColumnIndex].Type is VectorDataViewType vt && vt.ItemType == BooleanDataViewType.Instance)).ToArray(); + var booleanFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature && !numericFeatures.Contains(c)); var catalogFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.CategoricalFeature); var imagePathFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.ImagePath); var textFeatureColumnNames = textFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray(); diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_creditapproval_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_creditapproval_test.approved.txt new file mode 100644 index 0000000000..c2bea3ba98 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_creditapproval_test.approved.txt @@ -0,0 +1,86 @@ +{ + "schema": "e0 * e1 * (e2 \u002B e3) * e4", + "currentSchema": "e0 * e1 * e2 * e4", + "estimators": { + "e0": { + "estimatorType": "ReplaceMissingValues", + "parameter": { + "OutputColumnNames": [ + "Features" + ], + "InputColumnNames": [ + "Features" + ] + } + }, + "e1": { + "estimatorType": "ConvertType", + "parameter": { + "OutputColumnNames": [ + "BooleanFeatures" + ], + "InputColumnNames": [ + "BooleanFeatures" + ] + } + }, + "e2": { + "estimatorType": "OneHotEncoding", + "parameter": { + "OutputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ], + "InputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ] + } + }, + "e3": { + "estimatorType": "OneHotHashEncoding", + "parameter": { + "OutputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ], + "InputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ] + } + }, + "e4": { + "estimatorType": "Concatenate", + "parameter": { + "InputColumnNames": [ + "Features", + "A1", + "A4", + "A5", + "A6", + "A7", + "A13", + "BooleanFeatures" + ], + "OutputColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt index 3a57617a98..fe234e918c 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * (e2 \u002B e3) * e4 * e5", + "currentSchema": "e0 * e1 * e2 * e4 * e5", "estimators": { "e0": { "estimatorType": "ReplaceMissingValues", diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs index 839cf0ffe0..84365449fe 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs @@ -74,6 +74,18 @@ public void AutoFeaturizer_newspaperchurn_test() Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions)); } + [Fact] + [UseReporter(typeof(DiffReporter))] + [UseApprovalSubdirectory("ApprovalTests")] + public void AutoFeaturizer_creditapproval_test() + { + // this test verify if auto featurizer can convert vector column to vector. + var context = new MLContext(1); + var dataset = DatasetUtil.GetCreditApprovalDataView(); + var pipeline = context.Auto().Featurizer(dataset, excludeColumns: new[] { "A16" }); + Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions)); + } + [Fact] [UseReporter(typeof(DiffReporter))] [UseApprovalSubdirectory("ApprovalTests")] diff --git a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs index 555a06521f..7eeead3bc9 100644 --- a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs +++ b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs @@ -100,6 +100,15 @@ public static IDataView GetNewspaperChurnDataView() return _newspaperChurnDataView; } + public static IDataView GetCreditApprovalDataView() + { + var context = new MLContext(1); + var file = GetDataPath(@"creditapproval_train.csv"); + var columnInferenceResult = context.Auto().InferColumns(file, "A16"); + var textLoader = context.Data.CreateTextLoader(columnInferenceResult.TextLoaderOptions); + return textLoader.Load(file); + } + public static IDataView GetTaxiFareTestDataView() { if (_taxiFareTestDataView == null) diff --git a/test/data/creditapproval_train.csv b/test/data/creditapproval_train.csv new file mode 100644 index 0000000000..12c7aabba2 --- /dev/null +++ b/test/data/creditapproval_train.csv @@ -0,0 +1,553 @@ +A1,A2,A3,A4,A5,A6,A7,A8,A9,A10,A11,A12,A13,A14,A15,A16 +a,?,1.5,u,g,ff,ff,0.0,f,t,2,t,g,00200,105,- +a,46.00,4.0,u,g,j,j,0.0,t,f,0,f,g,00100,960,+ +b,20.00,0.0,u,g,d,v,0.5,f,f,0,f,g,00144,0,- +b,47.33,6.5,u,g,c,v,1.0,f,f,0,t,g,00000,228,- +b,19.17,0.0,y,p,m,bb,0.0,f,f,0,t,s,00500,1,+ +b,24.33,6.625,y,p,d,v,5.5,t,f,0,t,s,00100,0,+ +b,21.58,0.79,y,p,cc,v,0.665,f,f,0,f,g,00160,0,- +b,25.00,12.5,u,g,aa,v,3.0,t,f,0,t,s,00020,0,+ +b,21.92,0.5,u,g,c,v,0.125,f,f,0,f,g,00360,0,- +b,20.00,11.045,u,g,c,v,2.0,f,f,0,t,g,00136,0,- +a,23.50,1.5,u,g,w,v,0.875,f,f,0,t,g,00160,0,- +a,40.83,10.0,u,g,q,h,1.75,t,f,0,f,g,00029,837,+ +b,22.67,0.165,u,g,c,j,2.25,f,f,0,t,s,00000,0,+ +b,21.67,1.165,y,p,k,v,2.5,t,t,1,f,g,00180,20,- +a,30.58,10.665,u,g,q,h,0.085,f,t,12,t,g,00129,3,- +b,20.17,9.25,u,g,c,v,1.665,t,t,3,t,g,00040,28,+ +b,27.83,1.0,y,p,d,h,3.0,f,f,0,f,g,00176,537,- +a,16.08,0.335,u,g,ff,ff,0.0,f,t,1,f,g,00160,126,- +b,37.33,2.665,u,g,cc,v,0.165,f,f,0,t,g,00000,501,- +b,29.83,2.04,y,p,x,h,0.04,f,f,0,f,g,00128,1,- +a,16.00,0.165,u,g,aa,v,1.0,f,t,2,t,g,00320,1,- +b,34.75,2.5,u,g,cc,bb,0.5,f,f,0,f,g,00348,0,- +a,21.75,1.75,y,p,j,j,0.0,f,f,0,f,g,00160,0,- +b,21.92,0.54,y,p,x,v,0.04,t,t,1,t,g,00840,59,+ +b,23.92,0.585,y,p,cc,h,0.125,f,f,0,f,g,00240,1,- +?,40.83,3.5,u,g,i,bb,0.5,f,f,0,f,s,01160,0,- +b,40.25,21.5,u,g,e,z,20.0,t,t,11,f,g,00000,1200,+ +b,34.25,3.0,u,g,cc,h,7.415,t,f,0,t,g,00000,0,+ +a,22.50,0.415,u,g,i,v,0.335,f,f,0,t,s,00144,0,- +b,27.67,0.75,u,g,q,h,0.165,f,f,0,t,g,00220,251,- +b,36.25,5.0,u,g,c,bb,2.5,t,t,6,f,g,00000,367,+ +b,?,4.0,u,g,x,v,5.0,t,t,3,t,g,00290,2279,+ +b,39.92,0.54,y,p,aa,v,0.5,t,t,3,f,g,00200,1000,+ +a,19.75,0.75,u,g,c,v,0.795,t,t,5,t,g,00140,5,- +b,38.67,0.21,u,g,k,v,0.085,t,f,0,t,g,00280,0,+ +a,20.67,1.835,u,g,q,v,2.085,t,t,5,f,g,00220,2503,+ +b,60.08,14.5,u,g,ff,ff,18.0,t,t,15,t,g,00000,1000,+ +b,35.25,3.165,u,g,x,h,3.75,t,f,0,t,g,00680,0,- +b,18.50,2.0,u,g,i,v,1.5,t,t,2,f,g,00120,300,+ +a,47.42,8.0,u,g,e,bb,6.5,t,t,6,f,g,00375,51100,+ +b,21.83,11.0,u,g,x,v,0.29,t,t,6,f,g,00121,0,+ +a,38.58,5.0,u,g,cc,v,13.5,t,f,0,t,g,00980,0,- +b,30.75,1.585,u,g,d,v,0.585,f,f,0,t,s,00000,0,- +b,43.08,0.375,y,p,c,v,0.375,t,t,8,t,g,00300,162,+ +b,34.17,2.75,u,g,i,bb,2.5,f,f,0,t,g,00232,200,- +b,64.08,20.0,u,g,x,h,17.5,t,t,9,t,g,00000,1000,+ +a,38.92,1.665,u,g,aa,v,0.25,f,f,0,f,g,00000,390,- +b,34.00,5.085,y,p,i,bb,1.085,f,f,0,t,g,00480,0,- +b,34.08,6.5,u,g,aa,v,0.125,t,f,0,t,g,00443,0,- +b,22.75,11.5,u,g,i,v,0.415,f,f,0,f,g,00000,0,- +a,18.17,10.0,y,p,q,h,0.165,f,f,0,f,g,00340,0,- +b,62.50,12.75,y,p,c,h,5.0,t,f,0,f,g,00112,0,- +b,30.33,0.5,u,g,d,h,0.085,f,f,0,t,s,00252,0,- +b,22.08,0.83,u,g,c,h,2.165,f,f,0,t,g,00128,0,+ +a,20.42,0.835,u,g,q,v,1.585,t,t,1,f,g,00000,0,+ +a,22.67,7.0,u,g,c,v,0.165,f,f,0,f,g,00160,0,- +b,30.17,0.5,u,g,c,v,1.75,t,t,11,f,g,00032,540,+ +b,26.92,2.25,u,g,i,bb,0.5,f,f,0,t,g,00640,4000,- +b,48.33,12.0,u,g,m,v,16.0,t,f,0,f,s,00110,0,+ +a,15.75,0.375,u,g,c,v,1.0,f,f,0,f,g,00120,18,- +b,42.00,9.79,u,g,x,h,7.96,t,t,8,f,g,00000,0,+ +a,50.08,12.54,u,g,aa,v,2.29,t,t,3,t,g,00156,0,+ +b,20.42,7.0,u,g,c,v,1.625,t,t,3,f,g,00200,1391,+ +a,19.17,0.585,y,p,aa,v,0.585,t,f,0,t,g,00160,0,- +b,16.17,0.04,u,g,c,v,0.04,f,f,0,f,g,00000,0,+ +a,35.75,0.915,u,g,aa,v,0.75,t,t,4,f,g,00000,1583,+ +b,39.33,5.875,u,g,cc,h,10.0,t,t,14,t,g,00399,0,+ +b,23.33,11.625,y,p,w,v,0.835,t,f,0,t,g,00160,300,+ +b,48.58,6.5,u,g,q,h,6.0,t,f,0,t,g,00350,0,+ +a,24.83,4.5,u,g,w,v,1.0,f,f,0,t,g,00360,6,- +a,22.50,8.46,y,p,x,v,2.46,f,f,0,f,g,00164,0,+ +a,24.50,0.5,u,g,q,h,1.5,t,f,0,f,g,00280,824,+ +b,36.50,4.25,u,g,q,v,3.5,f,f,0,f,g,00454,50,- +a,27.67,1.5,u,g,m,v,2.0,t,f,0,f,s,00368,0,- +b,23.42,1.0,u,g,c,v,0.5,f,f,0,t,s,00280,0,- +b,21.08,10.085,y,p,e,h,1.25,f,f,0,f,g,00260,0,- +b,21.25,1.5,u,g,w,v,1.5,f,f,0,f,g,00150,8,+ +b,25.75,0.75,u,g,c,bb,0.25,t,f,0,f,g,00349,23,+ +b,39.92,6.21,u,g,q,v,0.04,t,t,1,f,g,00200,300,+ +b,25.83,12.835,u,g,cc,v,0.5,f,f,0,f,g,00000,2,- +a,19.67,0.21,u,g,q,h,0.29,t,t,11,f,g,00080,99,+ +a,22.42,11.25,y,p,x,h,0.75,t,t,4,f,g,00000,321,+ +b,19.42,7.25,u,g,m,v,0.04,f,t,1,f,g,00100,1,- +b,22.67,0.75,u,g,i,v,1.585,f,t,1,t,g,00400,9,- +b,20.42,1.835,u,g,c,v,2.25,t,t,1,f,g,00100,150,+ +a,30.25,5.5,u,g,k,v,5.5,f,f,0,t,s,00100,0,- +b,56.75,12.25,u,g,m,v,1.25,t,t,4,t,g,00200,0,+ +a,30.50,6.5,u,g,c,bb,4.0,t,t,7,t,g,00000,3065,+ +b,29.25,14.79,u,g,aa,v,5.04,t,t,5,t,g,00168,0,+ +b,35.17,25.125,u,g,x,h,1.625,t,t,1,t,g,00515,500,+ +a,20.50,11.835,u,g,c,h,6.0,t,f,0,f,g,00340,0,+ +a,18.83,9.54,u,g,aa,v,0.085,t,f,0,f,g,00100,0,+ +b,56.42,28.0,y,p,c,v,28.5,t,t,40,f,g,00000,15,+ +a,57.08,0.335,u,g,i,bb,1.0,t,f,0,t,g,00252,2197,- +b,32.33,7.5,u,g,e,bb,1.585,t,f,0,t,s,00420,0,- +a,20.67,3.0,u,g,q,v,0.165,t,t,3,f,g,00100,6,+ +a,60.58,16.5,u,g,q,v,11.0,t,f,0,t,g,00021,10561,+ +b,41.58,1.75,u,g,k,v,0.21,t,f,0,f,g,00160,0,- +b,41.17,1.335,u,g,d,v,0.165,f,f,0,f,g,00168,0,- +a,21.08,5.0,y,p,ff,ff,0.0,f,f,0,f,g,00000,0,- +a,69.50,6.0,u,g,ff,ff,0.0,f,f,0,f,s,00000,0,- +a,18.08,0.375,l,gg,cc,ff,10.0,f,f,0,t,s,00300,0,+ +?,24.50,12.75,u,g,c,bb,4.75,t,t,2,f,g,00073,444,+ +a,24.92,1.25,u,g,ff,ff,0.0,t,f,0,f,g,00080,0,- +b,34.17,1.54,u,g,cc,v,1.54,t,t,1,t,g,00520,50000,+ +b,45.17,1.5,u,g,c,v,2.5,t,f,0,t,g,00140,0,- +b,23.92,0.665,u,g,c,v,0.165,f,f,0,f,g,00100,0,+ +b,57.83,7.04,u,g,m,v,14.0,t,t,6,t,g,00360,1332,+ +b,50.75,0.585,u,g,ff,ff,0.0,f,f,0,f,g,00145,0,- +b,54.58,9.415,u,g,ff,ff,14.415,t,t,11,t,g,00030,300,+ +b,32.25,14.0,y,p,ff,ff,0.0,f,t,2,f,g,00160,1,- +b,41.75,0.96,u,g,x,v,2.5,t,f,0,f,g,00510,600,+ +b,33.17,1.04,u,g,r,h,6.5,t,f,0,t,g,00164,31285,+ +b,35.25,16.5,y,p,c,v,4.0,t,f,0,f,g,00080,0,- +a,21.67,11.5,y,p,j,j,0.0,t,t,11,t,g,00000,0,+ +b,69.17,9.0,u,g,ff,ff,4.0,f,t,1,f,g,00070,6,- +a,47.42,3.0,u,g,x,v,13.875,t,t,2,t,g,00519,1704,+ +b,22.58,10.04,u,g,x,v,0.04,t,t,9,f,g,00060,396,+ +b,23.25,12.625,u,g,c,v,0.125,f,t,2,f,g,00000,5552,- +b,32.83,2.5,u,g,cc,h,2.75,t,t,6,f,g,00160,2072,+ +b,42.75,4.085,u,g,aa,v,0.04,f,f,0,f,g,00108,100,- +b,23.08,2.5,u,g,ff,ff,0.085,f,f,0,t,g,00100,4208,- +b,23.08,11.5,u,g,i,v,3.5,t,t,9,f,g,00056,742,+ +a,22.67,0.75,u,g,c,v,2.0,f,t,2,t,g,00200,394,- +b,51.83,2.04,y,p,ff,ff,1.5,f,f,0,f,g,00120,1,- +b,?,0.375,u,g,d,v,0.875,t,f,0,t,s,00928,0,- +b,36.67,2.0,u,g,i,v,0.25,f,f,0,t,g,00221,0,- +b,27.58,3.25,y,p,q,h,5.085,f,t,2,t,g,00369,1,- +b,29.58,4.5,u,g,w,v,7.5,t,t,2,t,g,00330,0,+ +b,28.92,15.0,u,g,c,h,5.335,t,t,11,f,g,00000,2283,+ +b,40.92,2.25,y,p,x,h,10.0,t,f,0,t,g,00176,0,- +b,35.58,0.75,u,g,k,v,1.5,f,f,0,t,g,00231,0,- +b,23.17,0.0,u,g,cc,v,0.085,t,f,0,f,g,00000,0,+ +a,25.25,12.5,u,g,d,v,1.0,f,f,0,t,g,00180,1062,- +a,24.75,3.0,u,g,q,h,1.835,t,t,19,f,g,00000,500,+ +a,41.17,6.5,u,g,q,v,0.5,t,t,3,t,g,00145,0,+ +b,28.00,2.0,u,g,k,h,4.165,t,t,2,t,g,00181,0,+ +a,28.17,0.375,u,g,q,v,0.585,t,t,4,f,g,00080,0,+ +b,24.58,13.5,y,p,ff,ff,0.0,f,f,0,f,g,?,0,- +b,29.17,3.5,u,g,w,v,3.5,t,t,3,t,g,00329,0,- +b,39.83,0.5,u,g,m,v,0.25,t,f,0,f,s,00288,0,- +b,26.00,1.0,u,g,q,v,1.75,t,f,0,t,g,00280,0,+ +b,53.92,9.625,u,g,e,v,8.665,t,t,5,f,g,00000,0,+ +b,34.75,15.0,u,g,r,n,5.375,t,t,9,t,g,00000,134,+ +b,16.50,0.125,u,g,c,v,0.165,f,f,0,f,g,00132,0,- +b,25.42,0.54,u,g,w,v,0.165,f,t,1,f,g,00272,444,- +b,16.25,0.0,y,p,aa,v,0.25,f,f,0,f,g,00060,0,- +b,31.83,2.5,u,g,aa,v,7.5,t,f,0,t,g,00523,0,- +b,21.83,0.25,u,g,d,h,0.665,t,f,0,t,g,00000,0,+ +b,48.08,3.75,u,g,i,bb,1.0,f,f,0,f,g,00100,2,- +b,24.50,13.335,y,p,aa,v,0.04,f,f,0,t,g,00120,475,- +b,33.67,2.165,u,g,c,v,1.5,f,f,0,f,p,00120,0,- +b,?,0.04,y,p,d,v,4.25,f,f,0,t,g,00460,0,- +b,31.67,16.165,u,g,d,v,3.0,t,t,9,f,g,00250,730,+ +a,27.25,0.29,u,g,m,h,0.125,f,t,1,t,g,00272,108,- +b,39.00,5.0,u,g,cc,v,3.5,t,t,10,t,g,00000,0,+ +b,42.08,1.04,u,g,w,v,5.0,t,t,6,t,g,00500,10000,+ +a,21.25,2.335,u,g,i,bb,0.5,t,t,4,f,s,00080,0,+ +a,22.67,0.79,u,g,i,v,0.085,f,f,0,f,g,00144,0,- +a,30.42,1.375,u,g,w,h,0.04,f,t,3,f,g,00000,33,- +a,47.25,0.75,u,g,q,h,2.75,t,t,1,f,g,00333,892,+ +b,22.92,3.165,y,p,c,v,0.165,f,f,0,f,g,00160,1058,- +b,35.00,2.5,u,g,i,v,1.0,f,f,0,t,g,00210,0,- +b,16.92,0.335,y,p,k,v,0.29,f,f,0,f,s,00200,0,- +a,33.25,3.0,y,p,aa,v,2.0,f,f,0,f,g,00180,0,- +b,20.67,0.835,y,p,c,v,2.0,f,f,0,t,s,00240,0,- +b,24.58,1.25,u,g,c,v,0.25,f,f,0,f,g,00110,0,- +b,20.50,2.415,u,g,c,v,2.0,t,t,11,t,g,00200,3000,+ +a,19.50,0.165,u,g,q,v,0.04,f,f,0,t,g,00380,0,- +b,48.17,3.5,u,g,aa,v,3.5,t,f,0,f,s,00230,0,+ +a,19.58,0.665,u,g,w,v,1.665,f,f,0,f,g,00220,5,- +b,15.17,7.0,u,g,e,v,1.0,f,f,0,f,g,00600,0,- +b,22.67,10.5,u,g,q,h,1.335,t,f,0,f,g,00100,0,+ +b,24.08,9.0,u,g,aa,v,0.25,f,f,0,t,g,00000,0,- +b,32.00,1.75,y,p,e,h,0.04,t,f,0,t,g,00393,0,+ +b,43.17,5.0,u,g,i,bb,2.25,f,f,0,t,g,00141,0,- +b,20.08,0.25,u,g,q,v,0.125,f,f,0,f,g,00200,0,- +b,21.83,1.54,u,g,k,v,0.085,f,f,0,t,g,00356,0,- +a,39.08,4.0,u,g,c,v,3.0,f,f,0,f,g,00480,0,- +b,18.58,5.71,u,g,d,v,0.54,f,f,0,f,g,00120,0,- +a,17.33,9.5,u,g,aa,v,1.75,f,t,10,t,g,00000,10,- +b,27.83,4.0,y,p,i,h,5.75,t,t,2,t,g,00075,0,- +b,20.67,1.25,y,p,c,h,1.375,t,t,3,t,g,00140,210,- +b,29.58,4.75,u,g,m,v,2.0,f,t,1,t,g,00460,68,- +b,35.17,2.5,u,g,k,v,4.5,t,t,7,f,g,00150,1270,+ +b,32.75,2.335,u,g,d,h,5.75,f,f,0,t,g,00292,0,- +b,?,5.0,y,p,aa,v,8.5,t,f,0,f,g,00000,0,- +a,22.92,11.585,u,g,cc,v,0.04,t,f,0,f,g,00080,1349,+ +a,49.00,1.5,u,g,j,j,0.0,t,f,0,t,g,00100,27,- +b,22.83,3.0,u,g,m,v,1.29,t,t,1,f,g,00260,800,+ +a,33.08,1.625,u,g,d,v,0.54,f,f,0,t,g,00000,0,- +b,?,10.5,u,g,x,v,6.5,t,f,0,f,g,00000,0,+ +b,39.42,1.71,y,p,m,v,0.165,f,f,0,f,s,00400,0,- +b,80.25,5.5,u,g,?,?,0.54,t,f,0,f,g,00000,340,- +b,36.33,2.125,y,p,w,v,0.085,t,t,1,f,g,00050,1187,+ +a,31.25,3.75,u,g,cc,h,0.625,t,t,9,t,g,00181,0,+ +b,57.42,8.5,u,g,e,h,7.0,t,t,3,f,g,00000,0,+ +b,39.58,13.915,u,g,w,v,8.625,t,t,6,t,g,00070,0,+ +b,29.67,1.415,u,g,w,h,0.75,t,t,1,f,g,00240,100,+ +b,44.25,0.5,u,g,m,v,10.75,t,f,0,f,s,00400,0,- +a,32.33,0.54,u,g,cc,v,0.04,t,f,0,f,g,00440,11177,+ +b,39.50,4.25,u,g,c,bb,6.5,t,t,16,f,g,00117,1210,+ +?,42.25,1.75,y,p,?,?,0.0,f,f,0,t,g,00150,1,- +a,36.00,1.0,u,g,c,v,2.0,t,t,11,f,g,00000,456,+ +b,29.92,1.835,u,g,c,h,4.335,t,f,0,f,g,00260,200,+ +a,37.33,2.5,u,g,i,h,0.21,f,f,0,f,g,00260,246,- +a,22.83,2.29,u,g,q,h,2.29,t,t,7,t,g,00140,2384,+ +a,18.42,9.25,u,g,q,v,1.21,t,t,4,f,g,00060,540,+ +b,17.08,0.085,y,p,c,v,0.04,f,f,0,f,g,00140,722,- +b,22.50,11.5,y,p,m,v,1.5,f,f,0,t,g,00000,4000,- +b,43.25,25.21,u,g,q,h,0.21,t,t,1,f,g,00760,90,- +b,17.08,0.25,u,g,q,v,0.335,f,t,4,f,g,00160,8,- +b,26.67,1.75,y,p,c,v,1.0,t,t,5,t,g,00160,5777,+ +a,26.08,8.665,u,g,aa,v,1.415,t,f,0,f,g,00160,150,+ +b,20.00,7.0,u,g,c,v,0.5,f,f,0,f,g,00000,0,- +b,25.67,3.25,u,g,c,h,2.29,f,t,1,t,g,00416,21,- +a,?,3.5,u,g,d,v,3.0,t,f,0,t,g,00300,0,- +a,27.33,1.665,u,g,ff,ff,0.0,f,f,0,f,g,00340,1,- +b,27.83,1.5,u,g,w,v,2.25,f,t,1,t,g,00100,3,- +?,29.75,0.665,u,g,w,v,0.25,f,f,0,t,g,00300,0,- +b,21.50,9.75,u,g,c,v,0.25,t,f,0,f,g,00140,0,- +a,27.42,14.5,u,g,x,h,3.085,t,t,1,f,g,00120,11,+ +b,36.67,4.415,y,p,k,v,0.25,t,t,10,t,g,00320,0,+ +a,20.83,0.5,y,p,e,dd,1.0,f,f,0,f,g,00260,0,- +a,58.42,21.0,u,g,i,bb,10.0,t,t,13,f,g,00000,6700,+ +b,36.17,0.42,y,p,w,v,0.29,f,f,0,t,g,00309,2,- +b,32.33,2.5,u,g,c,v,1.25,f,f,0,t,g,00280,0,- +b,19.00,1.75,y,p,c,v,2.335,f,f,0,t,g,00112,6,- +a,24.33,2.5,y,p,i,bb,4.5,f,f,0,f,g,00200,456,- +b,33.58,2.75,u,g,m,v,4.25,t,t,6,f,g,00204,0,+ +?,32.25,1.5,u,g,c,v,0.25,f,f,0,t,g,00372,122,- +b,30.17,1.085,y,p,c,v,0.04,f,f,0,f,g,00170,179,- +a,36.75,5.125,u,g,e,v,5.0,t,f,0,t,g,00000,4000,+ +a,27.67,2.04,u,g,w,v,0.25,f,f,0,t,g,00180,50,- +b,39.17,2.5,y,p,i,h,10.0,f,f,0,t,s,00200,0,- +b,42.50,4.915,y,p,w,v,3.165,t,f,0,t,g,00052,1442,+ +a,47.75,8.0,u,g,c,v,7.875,t,t,6,t,g,00000,1260,+ +a,57.58,2.0,u,g,ff,ff,6.5,f,t,1,f,g,00000,10,- +b,65.42,11.0,u,g,e,z,20.0,t,t,7,t,g,00022,0,+ +b,26.17,12.5,y,p,k,h,1.25,f,f,0,t,g,00000,17,- +a,41.58,1.04,u,g,aa,v,0.665,f,f,0,f,g,00240,237,- +b,76.75,22.29,u,g,e,z,12.75,t,t,1,t,g,00000,109,+ +b,27.25,0.625,u,g,aa,v,0.455,t,f,0,t,g,00200,0,- +a,33.25,2.5,y,p,c,v,2.5,f,f,0,t,g,00000,2,- +b,34.08,2.5,u,g,c,v,1.0,f,f,0,f,g,00460,16,- +b,39.17,1.625,u,g,c,v,1.5,t,t,10,f,g,00186,4700,+ +a,24.50,1.75,y,p,c,v,0.165,f,f,0,f,g,00132,0,- +a,20.75,10.25,u,g,q,v,0.71,t,t,2,t,g,00049,0,+ +b,33.17,3.165,y,p,x,v,3.165,t,t,3,t,g,00380,0,+ +a,15.83,7.625,u,g,q,v,0.125,f,t,1,t,g,00000,160,- +b,20.25,9.96,u,g,e,dd,0.0,t,f,0,f,g,00000,0,+ +b,24.75,0.54,u,g,m,v,1.0,f,f,0,t,g,00120,1,- +b,52.33,1.375,y,p,c,h,9.46,t,f,0,t,g,00200,100,- +b,18.83,0.0,u,g,q,v,0.665,f,f,0,f,g,00160,1,- +a,18.25,10.0,u,g,w,v,1.0,f,t,1,f,g,00120,1,- +b,22.17,12.125,u,g,c,v,3.335,f,t,2,t,g,00180,173,- +b,21.42,0.75,y,p,r,n,0.75,f,f,0,t,g,00132,2,- +a,22.42,5.665,u,g,q,v,2.585,t,t,7,f,g,00129,3257,+ +a,25.17,2.875,u,g,x,h,0.875,t,f,0,f,g,00360,0,+ +b,23.00,0.75,u,g,m,v,0.5,f,f,0,t,s,00320,0,- +b,23.08,2.5,u,g,c,v,1.085,t,t,11,t,g,00060,2184,+ +b,20.67,5.29,u,g,q,v,0.375,t,t,1,f,g,00160,0,- +b,34.83,2.5,y,p,w,v,3.0,f,f,0,f,s,00200,0,- +b,19.42,6.5,u,g,w,h,1.46,t,t,7,f,g,00080,2954,+ +b,40.92,0.835,u,g,ff,ff,0.0,t,f,0,f,g,00130,1,- +a,28.67,1.04,u,g,c,v,2.5,t,t,5,t,g,00300,1430,+ +b,26.17,0.25,u,g,i,bb,0.0,t,f,0,t,g,00000,0,+ +b,34.00,4.5,u,g,aa,v,1.0,t,f,0,t,g,00240,0,- +b,41.00,0.04,u,g,e,v,0.04,f,t,1,f,s,00560,0,+ +b,52.50,6.5,u,g,k,v,6.29,t,t,15,f,g,00000,11202,+ +b,34.92,5.0,u,g,x,h,7.5,t,t,6,t,g,00000,1000,+ +b,19.50,9.585,u,g,aa,v,0.79,f,f,0,f,g,00080,350,- +b,42.83,4.625,u,g,q,v,4.58,t,f,0,f,s,00000,0,+ +b,30.83,0.0,u,g,w,v,1.25,t,t,1,f,g,00202,0,+ +b,17.08,3.29,u,g,i,v,0.335,f,f,0,t,g,00140,2,- +b,33.17,1.0,u,g,x,v,0.75,t,t,7,t,g,00340,4071,+ +b,22.08,2.335,u,g,k,v,0.75,f,f,0,f,g,00180,0,- +b,23.58,1.79,u,g,c,v,0.54,f,f,0,t,g,00136,1,- +b,37.75,7.0,u,g,q,h,11.5,t,t,7,t,g,00300,5,- +b,19.67,0.375,u,g,q,v,2.0,t,t,2,t,g,00080,0,+ +b,23.50,2.75,u,g,ff,ff,4.5,f,f,0,f,g,00160,25,- +b,25.33,0.58,u,g,c,v,0.29,t,t,7,t,g,00096,5124,+ +b,30.67,2.5,u,g,cc,h,2.25,f,f,0,t,s,00340,0,- +b,19.58,0.585,u,g,ff,ff,0.0,f,t,3,f,g,00350,769,- +b,27.25,1.665,u,g,cc,h,5.085,t,t,9,f,g,00399,827,+ +b,19.67,10.0,y,p,k,h,0.835,t,f,0,t,g,00140,0,- +b,32.42,2.165,y,p,k,ff,0.0,f,f,0,f,g,00120,0,- +a,25.58,0.0,?,?,?,?,0.0,f,f,0,f,p,?,0,+ +a,29.58,1.75,y,p,k,v,1.25,f,f,0,t,g,00280,0,- +b,59.67,1.54,u,g,q,v,0.125,t,f,0,t,g,00260,0,+ +a,17.92,10.21,u,g,ff,ff,0.0,f,f,0,f,g,00000,50,- +a,41.00,2.04,y,p,q,h,0.125,t,t,23,t,g,00455,1236,+ +a,52.17,0.0,y,p,ff,ff,0.0,f,f,0,f,g,00000,0,- +b,28.25,5.04,y,p,c,bb,1.5,t,t,8,t,g,00144,7,+ +b,35.00,3.375,u,g,c,h,8.29,f,f,0,t,g,00000,0,- +a,20.33,10.0,u,g,c,h,1.0,t,t,4,f,g,00050,1465,+ +a,25.00,11.0,y,p,aa,v,4.5,t,f,0,f,g,00120,0,- +b,34.25,1.75,u,g,w,bb,0.25,t,f,0,t,g,00163,0,- +b,22.67,2.54,y,p,c,h,2.585,t,f,0,f,g,00000,0,+ +b,36.75,0.125,y,p,c,v,1.5,f,f,0,t,g,00232,113,+ +b,37.50,1.75,y,p,c,bb,0.25,t,f,0,t,g,00164,400,- +b,29.67,0.75,y,p,c,v,0.04,f,f,0,f,g,00240,0,- +b,29.42,1.25,u,g,c,h,0.25,f,t,2,t,g,00400,108,- +b,?,4.0,y,p,i,v,0.085,f,f,0,t,g,00411,0,- +b,20.17,8.17,u,g,aa,v,1.96,t,t,14,f,g,00060,158,+ +a,24.75,13.665,u,g,q,h,1.5,f,f,0,f,g,00280,1,- +b,38.92,1.75,u,g,k,v,0.5,f,f,0,t,g,00300,2,- +a,52.83,15.0,u,g,c,v,5.5,t,t,14,f,g,00000,2200,+ +a,23.50,9.0,u,g,q,v,8.5,t,t,5,t,g,00120,0,+ +b,44.25,11.0,y,p,d,v,1.5,t,f,0,f,s,00000,0,- +a,46.08,3.0,u,g,c,v,2.375,t,t,8,t,g,00396,4159,+ +b,48.75,8.5,u,g,c,h,12.5,t,t,9,f,g,00181,1655,+ +b,29.42,1.25,u,g,w,v,1.75,f,f,0,f,g,00200,0,- +b,26.67,14.585,u,g,i,bb,0.0,f,f,0,t,g,00178,0,- +b,19.50,0.29,u,g,k,v,0.29,f,f,0,f,g,00280,364,- +a,38.75,1.5,u,g,ff,ff,0.0,f,f,0,f,g,00076,0,- +b,18.17,2.46,u,g,c,n,0.96,f,t,2,t,g,00160,587,- +b,43.00,0.29,y,p,cc,h,1.75,t,t,8,f,g,00100,375,+ +a,56.50,16.0,u,g,j,ff,0.0,t,t,15,f,g,00000,247,+ +b,22.00,7.835,y,p,i,bb,0.165,f,f,0,t,g,?,0,- +a,19.17,8.585,u,g,cc,h,0.75,t,t,7,f,g,00096,0,+ +b,56.00,12.5,u,g,k,h,8.0,t,f,0,t,g,00024,2028,+ +a,32.17,1.46,u,g,w,v,1.085,t,t,16,f,g,00120,2079,+ +a,16.33,0.21,u,g,aa,v,0.125,f,f,0,f,g,00200,1,- +?,33.17,2.25,y,p,cc,v,3.5,f,f,0,t,g,00200,141,- +a,38.25,6.0,u,g,k,v,1.0,t,f,0,t,g,00000,0,+ +b,48.75,26.335,y,p,ff,ff,0.0,t,f,0,t,g,00000,0,- +a,15.83,0.585,u,g,c,h,1.5,t,t,2,f,g,00100,0,+ +a,44.33,0.0,u,g,c,v,2.5,t,f,0,f,g,00000,0,+ +b,37.50,1.125,y,p,d,v,1.5,f,f,0,t,g,00431,0,+ +b,43.17,2.25,u,g,i,bb,0.75,t,f,0,f,g,00560,0,- +b,16.00,3.125,u,g,w,v,0.085,f,t,1,f,g,00000,6,- +b,33.17,3.04,y,p,c,h,2.04,t,t,1,t,g,00180,18027,+ +b,16.33,2.75,u,g,aa,v,0.665,f,t,1,f,g,00080,21,- +b,49.17,2.29,u,g,ff,ff,0.29,f,f,0,f,g,00200,3,- +b,18.08,6.75,y,p,m,v,0.04,f,f,0,f,g,00140,0,- +b,32.75,1.5,u,g,cc,h,5.5,t,t,3,t,g,00000,0,+ +b,47.67,2.5,u,g,m,bb,2.5,t,t,12,t,g,00410,2510,+ +a,23.00,1.835,u,g,j,j,0.0,f,t,1,f,g,00200,53,- +b,29.50,0.58,u,g,w,v,0.29,f,t,1,f,g,00340,2803,- +a,33.75,0.75,u,g,k,bb,1.0,t,t,3,t,g,00212,0,- +b,26.83,0.54,u,g,k,ff,0.0,f,f,0,f,g,00100,0,- +a,49.83,13.585,u,g,k,h,8.5,t,f,0,t,g,00000,0,+ +a,26.17,2.0,u,g,j,j,0.0,f,f,0,t,g,00276,1,- +b,44.33,0.5,u,g,i,h,5.0,t,f,0,t,g,00320,0,+ +a,53.33,0.165,u,g,ff,ff,0.0,f,f,0,t,s,00062,27,- +a,17.58,9.0,u,g,aa,v,1.375,t,f,0,t,g,00000,0,+ +b,21.17,0.875,y,p,c,h,0.25,f,f,0,f,g,00280,204,- +a,23.25,5.875,u,g,q,v,3.17,t,t,10,f,g,00120,245,+ +b,31.00,2.085,u,g,c,v,0.085,f,f,0,f,g,00300,0,- +a,33.67,0.375,u,g,cc,v,0.375,f,f,0,f,g,00300,44,+ +b,18.17,10.25,u,g,c,h,1.085,f,f,0,f,g,00320,13,- +b,25.17,3.0,u,g,c,v,1.25,f,t,1,f,g,00000,22,- +b,22.25,9.0,u,g,aa,v,0.085,f,f,0,f,g,00000,0,- +b,21.33,10.5,u,g,c,v,3.0,t,f,0,t,g,00000,0,+ +a,27.58,3.0,u,g,m,v,2.79,f,t,1,t,g,00280,10,- +b,32.33,3.5,u,g,k,v,0.5,f,f,0,t,g,00232,0,- +b,27.00,0.75,u,g,c,h,4.25,t,t,3,t,g,00312,150,+ +b,28.75,1.5,y,p,c,v,1.5,t,f,0,t,g,00000,225,- +b,47.67,0.29,u,g,c,bb,15.0,t,t,20,f,g,00000,15000,+ +b,18.25,0.165,u,g,d,v,0.25,f,f,0,t,s,00280,0,- +b,33.58,0.335,y,p,cc,v,0.085,f,f,0,f,g,00180,0,- +b,28.25,5.125,u,g,x,v,4.75,t,t,2,f,g,00420,7,+ +b,31.25,1.125,u,g,ff,ff,0.0,f,t,1,f,g,00096,19,- +a,?,11.25,u,g,ff,ff,0.0,f,f,0,f,g,?,5200,- +b,19.17,4.0,y,p,i,v,1.0,f,f,0,t,g,00360,1000,- +b,32.08,4.0,u,g,m,v,2.5,t,f,0,t,g,00360,0,+ +b,54.33,6.75,u,g,c,h,2.625,t,t,11,t,g,00000,284,+ +b,59.50,2.75,u,g,w,v,1.75,t,t,5,t,g,00060,58,+ +b,39.17,1.71,u,g,x,v,0.125,t,t,5,t,g,00480,0,+ +b,41.33,0.0,u,g,c,bb,15.0,t,f,0,f,g,00000,0,+ +b,28.25,0.875,u,g,m,v,0.96,t,t,3,t,g,00396,0,+ +b,41.50,1.54,u,g,i,bb,3.5,f,f,0,f,g,00216,0,+ +a,21.75,11.75,u,g,c,v,0.25,f,f,0,t,g,00180,0,- +a,22.92,1.25,u,g,q,v,0.25,f,f,0,t,g,00120,809,- +b,27.83,1.54,u,g,w,v,3.75,t,t,5,t,g,00100,3,+ +b,17.25,3.0,u,g,k,v,0.04,f,f,0,t,g,00160,40,- +b,36.08,2.54,u,g,ff,ff,0.0,f,f,0,f,g,00000,1000,- +b,21.17,0.25,y,p,c,h,0.25,f,f,0,f,g,00280,204,- +a,23.75,0.71,u,g,w,v,0.25,f,t,1,t,g,00240,4,- +a,16.50,1.25,u,g,q,v,0.25,f,t,1,f,g,00108,98,- +a,18.83,4.415,y,p,c,h,3.0,t,f,0,f,g,00240,0,+ +b,17.58,10.0,u,g,w,h,0.165,f,t,1,f,g,00120,1,- +b,58.58,2.71,u,g,c,v,2.415,f,f,0,t,g,00320,0,- +b,23.25,4.0,u,g,c,bb,0.25,t,f,0,t,g,00160,0,+ +a,47.00,13.0,u,g,i,bb,5.165,t,t,9,t,g,00000,0,+ +b,42.75,3.0,u,g,i,bb,1.0,t,f,0,f,g,00000,200,- +a,20.42,10.5,y,p,x,h,0.0,f,f,0,t,g,00154,32,- +a,25.75,0.5,u,g,c,h,0.875,t,f,0,t,g,00491,0,+ +b,24.42,2.0,u,g,e,dd,0.165,f,t,2,f,g,00320,1300,- +b,22.08,11.0,u,g,cc,v,0.665,t,f,0,f,g,00100,0,+ +a,23.00,11.75,u,g,x,h,0.5,t,t,2,t,g,00300,551,+ +a,68.67,15.0,u,g,e,z,0.0,t,t,14,f,g,00000,3376,+ +b,21.33,7.5,u,g,aa,v,1.415,t,t,1,f,g,00080,9800,+ +a,50.25,0.835,u,g,aa,v,0.5,f,f,0,t,g,00240,117,- +?,28.17,0.585,u,g,aa,v,0.04,f,f,0,f,g,00260,1004,- +b,33.00,2.5,y,p,w,v,7.0,f,f,0,t,g,00280,0,- +b,27.00,1.5,y,p,w,v,0.375,t,f,0,t,g,00260,1065,+ +b,48.50,4.25,u,g,m,v,0.125,t,f,0,t,g,00225,0,+ +b,23.92,1.5,u,g,d,h,1.875,t,t,6,f,g,00200,327,+ +a,18.92,9.0,u,g,aa,v,0.75,t,t,2,f,g,00088,591,+ +b,34.50,4.04,y,p,i,bb,8.5,t,t,7,t,g,00195,0,+ +a,28.08,15.0,y,p,e,z,0.0,t,f,0,f,g,00000,13212,+ +b,34.17,5.25,u,g,w,v,0.085,f,f,0,t,g,00290,6,+ +b,51.83,3.0,y,p,ff,ff,1.5,f,f,0,f,g,00180,4,- +b,25.92,0.875,u,g,k,v,0.375,f,t,2,t,g,00174,3,- +b,21.50,11.5,u,g,i,v,0.5,t,f,0,t,g,00100,68,- +b,31.83,0.04,y,p,m,v,0.04,f,f,0,f,g,00000,0,- +b,21.00,3.0,y,p,d,v,1.085,t,t,8,t,g,00160,1,+ +b,30.17,6.5,u,g,cc,v,3.125,t,t,8,f,g,00330,1200,+ +b,25.50,0.375,u,g,m,v,0.25,t,t,3,f,g,00260,15108,+ +b,17.92,0.205,u,g,aa,v,0.04,f,f,0,f,g,00280,750,- +b,36.67,3.25,u,g,q,h,9.0,t,f,0,t,g,00102,639,+ +a,55.75,7.08,u,g,k,h,6.75,t,t,3,t,g,00100,50,- +b,25.08,1.71,u,g,x,v,1.665,t,t,1,t,g,00395,20,+ +a,31.75,3.0,y,p,j,j,0.0,f,f,0,f,g,00160,20,- +b,22.67,1.585,y,p,w,v,3.085,t,t,6,f,g,00080,0,+ +a,21.92,11.665,u,g,k,h,0.085,f,f,0,f,g,00320,5,- +b,42.83,1.25,u,g,m,v,13.875,f,t,1,t,g,00352,112,- +a,56.83,4.25,y,p,ff,ff,5.0,f,f,0,t,g,00000,4,- +b,22.17,2.25,u,g,i,v,0.125,f,f,0,f,g,00160,10,- +b,18.83,3.54,y,p,ff,ff,0.0,f,f,0,t,g,00180,1,- +b,37.33,6.5,u,g,m,h,4.25,t,t,12,t,g,00093,0,+ +a,17.67,0.0,y,p,j,ff,0.0,f,f,0,f,g,00086,0,- +b,34.58,0.0,?,?,?,?,0.0,f,f,0,f,p,?,0,- +b,18.08,5.5,u,g,k,v,0.5,t,f,0,f,g,00080,0,+ +a,16.92,0.5,u,g,i,v,0.165,f,t,6,t,g,00240,35,- +b,51.33,10.0,u,g,i,bb,0.0,t,t,11,f,g,00000,1249,+ +b,25.00,12.0,u,g,k,v,2.25,t,t,2,t,g,00120,5,- +b,20.42,1.085,u,g,q,v,1.5,f,f,0,f,g,00108,7,- +b,23.75,0.415,y,p,c,v,0.04,f,t,2,f,g,00128,6,- +a,30.00,5.29,u,g,e,dd,2.25,t,t,5,t,g,00099,500,+ +b,44.83,7.0,y,p,c,v,1.625,f,f,0,f,g,00160,2,- +a,25.42,1.125,u,g,q,v,1.29,t,t,2,f,g,00200,0,- +a,19.58,0.665,y,p,c,v,1.0,f,t,1,f,g,02000,2,- +b,31.08,1.5,y,p,w,v,0.04,f,f,0,f,s,00160,0,- +a,20.75,10.335,u,g,cc,h,0.335,t,t,1,t,g,00080,50,+ +b,34.92,2.5,u,g,w,v,0.0,t,f,0,t,g,00239,200,+ +a,58.33,10.0,u,g,q,v,4.0,t,t,14,f,g,00000,1602,+ +a,22.67,0.335,u,g,q,v,0.75,f,f,0,f,s,00160,0,- +a,35.17,3.75,u,g,ff,ff,0.0,f,t,6,f,g,00000,200,- +b,28.67,9.335,u,g,q,h,5.665,t,t,6,f,g,00381,168,+ +b,39.50,1.625,u,g,c,v,1.5,f,f,0,f,g,00000,316,- +?,26.50,2.71,y,p,?,?,0.085,f,f,0,f,s,00080,0,- +b,32.25,0.165,y,p,c,h,3.25,t,t,1,t,g,00432,8000,+ +a,24.58,0.67,u,g,aa,h,1.75,t,f,0,f,g,00400,0,- +b,54.42,0.5,y,p,k,h,3.96,t,f,0,f,g,00180,314,+ +b,18.42,10.415,y,p,aa,v,0.125,t,f,0,f,g,00120,375,- +b,27.25,1.585,u,g,cc,h,1.835,t,t,12,t,g,00583,713,+ +a,26.92,13.5,u,g,q,h,5.0,t,t,2,f,g,00000,5000,+ +b,23.17,11.125,u,g,x,h,0.46,t,t,1,f,g,00100,0,+ +a,15.92,2.875,u,g,q,v,0.085,f,f,0,f,g,00120,0,- +b,16.08,0.75,u,g,c,v,1.75,t,t,5,t,g,00352,690,+ +b,31.58,0.75,y,p,aa,v,3.5,f,f,0,t,g,00320,0,- +a,23.58,11.5,y,p,k,h,3.0,f,f,0,t,g,00020,16,- +b,43.25,3.0,u,g,q,h,6.0,t,t,11,f,g,00080,0,+ +a,24.50,1.04,y,p,ff,ff,0.5,t,t,3,f,g,00180,147,- +b,23.50,3.165,y,p,k,v,0.415,f,t,1,t,g,00280,80,- +b,19.17,9.5,u,g,w,v,1.5,t,f,0,f,g,00120,2206,+ +b,74.83,19.0,y,p,ff,ff,0.04,f,t,2,f,g,00000,351,- +a,48.17,1.335,u,g,i,o,0.335,f,f,0,f,g,00000,120,- +b,25.17,6.0,u,g,c,v,1.0,t,t,3,f,g,00000,0,+ +b,32.92,2.5,u,g,aa,v,1.75,f,t,2,t,g,00720,0,- +b,22.25,0.46,u,g,k,v,0.125,f,f,0,t,g,00280,55,- +a,46.67,0.46,u,g,cc,h,0.415,t,t,11,t,g,00440,6,+ +b,25.25,1.0,u,g,aa,v,0.5,f,f,0,f,g,00200,0,- +b,23.08,0.0,u,g,k,v,1.0,f,t,11,f,s,00000,0,- +b,18.33,1.21,y,p,e,dd,0.0,f,f,0,f,g,00100,0,- +?,20.08,0.125,u,g,q,v,1.0,f,t,1,f,g,00240,768,+ +a,36.75,4.71,u,g,ff,ff,0.0,f,f,0,f,g,00160,0,- +b,20.00,1.25,y,p,k,v,0.125,f,f,0,f,g,00140,4,- +b,27.75,0.585,y,p,cc,v,0.25,t,t,2,f,g,00260,500,+ +b,22.75,11.0,u,g,q,v,2.5,t,t,7,t,g,00100,809,+ +b,26.17,0.835,u,g,cc,v,1.165,f,f,0,f,g,00100,0,- +b,22.50,0.125,y,p,k,v,0.125,f,f,0,f,g,00200,70,- +b,18.58,10.29,u,g,ff,ff,0.415,f,f,0,f,g,00080,0,- +b,27.58,2.04,y,p,aa,v,2.0,t,t,3,t,g,00370,560,+ +b,22.17,0.585,y,p,ff,ff,0.0,f,f,0,f,g,00100,0,- +b,28.17,0.125,y,p,k,v,0.085,f,f,0,f,g,00216,2100,- +b,23.25,1.5,u,g,q,v,2.375,t,t,3,t,g,00000,582,+ +b,?,3.0,y,p,i,bb,7.0,f,f,0,f,g,00000,1,- +b,28.58,3.625,u,g,aa,v,0.25,f,f,0,t,g,00100,0,- +a,60.92,5.0,u,g,aa,v,4.0,t,t,4,f,g,00000,99,+ +a,26.58,2.54,y,p,ff,ff,0.0,f,f,0,t,g,00180,60,- +b,30.58,2.71,y,p,m,v,0.125,f,f,0,t,s,00080,0,- +b,22.00,0.79,u,g,w,v,0.29,f,t,1,f,g,00420,283,- +a,40.33,7.54,y,p,q,h,8.0,t,t,14,f,g,00000,2300,+ +b,51.42,0.04,u,g,x,h,0.04,t,f,0,f,g,00000,3000,+ +b,63.33,0.54,u,g,c,v,0.585,t,t,3,t,g,00180,0,- +a,36.58,0.29,u,g,ff,ff,0.0,f,t,10,f,g,00200,18,- +a,29.50,1.085,y,p,x,v,1.0,f,f,0,f,g,00280,13,- +b,40.00,6.5,u,g,aa,bb,3.5,t,t,1,f,g,00000,500,+ +a,44.17,6.665,u,g,q,v,7.375,t,t,3,t,g,00000,0,+ +b,32.67,9.0,y,p,w,h,5.25,t,f,0,t,g,00154,0,+ +b,23.00,0.625,y,p,aa,v,0.125,t,f,0,f,g,00180,1,- +b,22.33,11.0,u,g,w,v,2.0,t,t,1,f,g,00080,278,+ +b,38.17,10.125,u,g,x,v,2.5,t,t,6,f,g,00520,196,+ +b,41.17,1.25,y,p,w,v,0.25,f,f,0,f,g,00000,195,- +b,16.33,4.085,u,g,i,h,0.415,f,f,0,t,g,00120,0,- +b,48.17,7.625,u,g,w,h,15.5,t,t,12,f,g,00000,790,+ +b,?,0.5,u,g,c,bb,0.835,t,f,0,t,s,00320,0,- +b,26.33,13.0,u,g,e,dd,0.0,f,f,0,t,g,00140,1110,- +b,33.67,1.25,u,g,w,v,1.165,f,f,0,f,g,00120,0,- +b,16.25,0.835,u,g,m,v,0.085,t,f,0,f,s,00200,0,- +b,35.75,2.415,u,g,w,v,0.125,f,t,2,f,g,00220,1,- +b,?,0.625,u,g,k,v,0.25,f,f,0,f,g,00380,2010,- +b,20.75,5.085,y,p,j,v,0.29,f,f,0,f,g,00140,184,- +a,24.75,12.5,u,g,aa,v,1.5,t,t,12,t,g,00120,567,+ +a,57.08,19.5,u,g,c,v,5.5,t,t,7,f,g,00000,3000,+ +b,24.83,2.75,u,g,c,v,2.25,t,t,6,f,g,?,600,+ +a,27.17,1.25,u,g,ff,ff,0.0,f,t,1,f,g,00092,300,- +b,48.58,0.205,y,p,k,v,0.25,t,t,11,f,g,00380,2732,+ +b,45.00,8.5,u,g,cc,h,14.0,t,t,1,t,g,00088,2000,+ +a,22.25,1.25,y,p,ff,ff,3.25,f,f,0,f,g,00280,0,- +b,47.83,4.165,u,g,x,bb,0.085,f,f,0,t,g,00520,0,- +a,28.42,3.5,u,g,w,v,0.835,t,f,0,f,s,00280,0,+ +a,51.92,6.5,u,g,i,bb,3.085,f,f,0,t,g,00073,0,- +b,36.33,3.79,u,g,w,v,1.165,t,f,0,t,g,00200,0,- +a,33.92,1.585,y,p,ff,ff,0.0,t,f,0,f,g,00320,0,- +a,18.92,9.25,y,p,c,v,1.0,t,t,4,t,g,00080,500,+ +b,23.75,12.0,u,g,c,v,2.085,f,f,0,f,s,00080,0,- +a,22.50,8.5,u,g,q,v,1.75,t,t,10,f,g,00080,990,- +b,29.25,13.0,u,g,d,h,0.5,f,f,0,f,g,00228,0,- +a,25.33,2.085,u,g,c,h,2.75,t,f,0,t,g,00360,1,- +b,22.58,1.5,y,p,aa,v,0.54,f,f,0,t,g,00120,67,- +a,17.92,0.54,u,g,c,v,1.75,f,t,1,t,g,00080,5,- +b,21.00,4.79,y,p,w,v,2.25,t,t,1,t,g,00080,300,+ +a,32.00,6.0,u,g,d,v,1.25,f,f,0,f,g,00272,0,- +b,39.58,5.0,u,g,ff,ff,0.0,f,t,2,f,g,00017,1,- +a,25.00,0.875,u,g,x,h,1.04,t,f,0,t,g,00160,5860,+ +b,24.58,12.5,u,g,w,v,0.875,t,f,0,t,g,00260,0,- +b,33.50,1.75,u,g,x,h,4.5,t,t,4,t,g,00253,857,+ +a,21.50,6.0,u,g,aa,v,2.5,t,t,3,f,g,00080,918,+ +b,26.25,1.54,u,g,w,v,0.125,f,f,0,f,g,00100,0,- +b,27.67,13.75,u,g,w,v,5.75,t,f,0,t,g,00487,500,+ +b,30.08,1.04,y,p,i,bb,0.5,t,t,10,t,g,00132,28,- +b,17.50,22.0,l,gg,ff,o,0.0,f,f,0,t,p,00450,100000,+ +b,31.25,2.835,u,g,ff,ff,0.0,f,t,5,f,g,00176,146,- +b,36.42,0.75,y,p,d,v,0.585,f,f,0,f,g,00240,3,- +b,32.42,3.0,u,g,d,v,0.165,f,f,0,t,g,00120,0,- +a,20.75,9.54,u,g,i,v,0.04,f,f,0,f,g,00200,1000,- +b,17.67,4.46,u,g,c,v,0.25,f,f,0,f,s,00080,0,- +b,28.67,14.5,u,g,d,v,0.125,f,f,0,f,g,00000,286,- +b,22.08,11.46,u,g,k,v,1.585,f,f,0,t,g,00100,1212,- +a,37.75,5.5,u,g,q,v,0.125,t,f,0,t,g,00228,0,+ +a,23.58,0.83,u,g,q,v,0.415,f,t,1,t,g,00200,11,- +a,28.58,3.75,u,g,c,v,0.25,f,t,1,t,g,00040,154,- +b,17.42,6.5,u,g,i,v,0.125,f,f,0,f,g,00060,100,- +b,26.75,4.5,y,p,c,bb,2.5,f,f,0,f,g,00200,1210,- +b,31.33,19.5,u,g,c,v,7.0,t,t,16,f,g,00000,5000,+ +a,20.08,1.25,u,g,c,v,0.0,f,f,0,f,g,00000,0,- +b,40.92,0.5,y,p,m,v,0.5,f,f,0,t,g,00130,0,- From daa3db71f226e2676ee28a4eb8ceff514c9a5451 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 21 Aug 2022 13:03:55 -0700 Subject: [PATCH 16/25] disable log_base for lgbm --- src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json b/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json index 2794e3c961..a0f6e1fd2f 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json +++ b/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json @@ -7,7 +7,7 @@ "type": "integer", "default": 4, "search_space": { - "log_base": true, + "log_base": false, "max": 32768, "min": 4 } @@ -39,7 +39,7 @@ "search_space": { "min": 4, "max": 32768, - "log_base": true + "log_base": false } }, { From dec85d863b5444ead9d44e2dd14a1d8a1142c5a8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 22 Aug 2022 12:09:29 -0700 Subject: [PATCH 17/25] clean up --- .../API/BinaryClassificationExperiment.cs | 2 +- .../API/MulticlassClassificationExperiment.cs | 2 +- .../AutoMLExperiment/AutoMLExperiment.cs | 87 ++++--------------- .../AutoMLExperiment/IDatasetManager.cs | 4 +- .../AutoMLExperiment/Runner/ITrialRunner.cs | 2 +- .../Runner/SweepablePipelineRunner.cs | 2 +- .../AutoMLExperimentTests.cs | 14 ++- 7 files changed, 34 insertions(+), 79 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 97c656a72d..e4ff36cc19 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -341,7 +341,7 @@ public BinaryClassificationRunner(MLContext context, IDatasetManager datasetMana _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); } - public TrialResult Run(TrialSettings settings, IServiceProvider provider) + public TrialResult Run(TrialSettings settings) { if (_metricManager is BinaryMetricManager metricManager) { diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 5ca2baa48a..6305e43594 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -348,7 +348,7 @@ public MulticlassClassificationRunner(MLContext context, IDatasetManager dataset _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); } - public TrialResult Run(TrialSettings settings, IServiceProvider provider) + public TrialResult Run(TrialSettings settings) { if (_metricManager is MultiClassMetricManager metricManager) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 640475788e..9e73e751b9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -44,7 +44,7 @@ public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) private void InitializeServiceCollection() { - _serviceCollection.TryAddSingleton((provider) => + _serviceCollection.TryAddTransient((provider) => { var context = new MLContext(_settings.Seed); _cts.Token.Register(() => @@ -91,30 +91,15 @@ public AutoMLExperiment AddSearchSpace(string key, SearchSpace.SearchSpace searc public AutoMLExperiment SetMonitor(TMonitor monitor) where TMonitor : class, IMonitor { - var descriptor = new ServiceDescriptor(typeof(IMonitor), monitor); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(ServiceDescriptor.Singleton(monitor)); return this; } public AutoMLExperiment SetMonitor() + where TMonitor : class, IMonitor { - var descriptor = new ServiceDescriptor(typeof(IMonitor), typeof(TMonitor), ServiceLifetime.Singleton); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(ServiceDescriptor.Singleton()); return this; } @@ -122,16 +107,7 @@ public AutoMLExperiment SetMonitor() public AutoMLExperiment SetMonitor(Func factory) where TMonitor : class, IMonitor { - var descriptor = ServiceDescriptor.Singleton(factory); - - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(ServiceDescriptor.Singleton(factory)); return this; } @@ -139,15 +115,7 @@ public AutoMLExperiment SetMonitor(Func fa public AutoMLExperiment SetTrialRunner(TTrialRunner runner) where TTrialRunner : class, ITrialRunner { - var descriptor = new ServiceDescriptor(typeof(ITrialRunner), runner); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(ServiceDescriptor.Singleton(runner)); return this; } @@ -155,16 +123,7 @@ public AutoMLExperiment SetTrialRunner(TTrialRunner runner) public AutoMLExperiment SetTrialRunner(Func factory) where TTrialRunner : class, ITrialRunner { - var descriptor = ServiceDescriptor.Singleton(factory); - - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddTransient(factory); return this; } @@ -172,15 +131,7 @@ public AutoMLExperiment SetTrialRunner(Func() where TTrialRunner : class, ITrialRunner { - var descriptor = new ServiceDescriptor(typeof(ITrialRunner), typeof(TTrialRunner), ServiceLifetime.Transient); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddTransient(); return this; } @@ -188,7 +139,13 @@ public AutoMLExperiment SetTrialRunner() public AutoMLExperiment SetTuner(TTuner proposer) where TTuner : class, ITuner { - var descriptor = new ServiceDescriptor(typeof(ITuner), proposer); + return this.SetTuner((service) => proposer); + } + + public AutoMLExperiment SetTuner(Func factory) + where TTuner : class, ITuner + { + var descriptor = ServiceDescriptor.Singleton(factory); if (_serviceCollection.Contains(descriptor)) { @@ -205,15 +162,7 @@ public AutoMLExperiment SetTuner(TTuner proposer) public AutoMLExperiment SetTuner() where TTuner : class, ITuner { - var descriptor = new ServiceDescriptor(typeof(ITuner), typeof(TTuner), ServiceLifetime.Singleton); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(); return this; } @@ -265,7 +214,7 @@ public async Task RunAsync(CancellationToken ct = default) try { - var trialResult = runner.Run(setting, serviceProvider); + var trialResult = runner.Run(setting); monitor?.ReportCompletedTrial(trialResult); tuner.Update(trialResult); @@ -283,7 +232,7 @@ public async Task RunAsync(CancellationToken ct = default) } catch (Exception ex) { - monitor.ReportFailTrial(setting, ex); + monitor?.ReportFailTrial(setting, ex); if (!_cts.IsCancellationRequested && _bestTrialResult == null) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index cd9533a606..ee752f3e07 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -26,14 +26,14 @@ internal interface ITrainTestDatasetManager IDataView TestDataset { get; set; } } - public class TrainTestDatasetManager : IDatasetManager, ITrainTestDatasetManager + internal class TrainTestDatasetManager : IDatasetManager, ITrainTestDatasetManager { public IDataView TrainDataset { get; set; } public IDataView TestDataset { get; set; } } - public class CrossValidateDatasetManager : IDatasetManager, ICrossValidateDatasetManager + internal class CrossValidateDatasetManager : IDatasetManager, ICrossValidateDatasetManager { public IDataView Dataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs index e43d018708..c19ae220b6 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs @@ -14,6 +14,6 @@ namespace Microsoft.ML.AutoML /// public interface ITrialRunner { - TrialResult Run(TrialSettings settings, IServiceProvider provider = null); + TrialResult Run(TrialSettings settings); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index d9c9f44f68..f869df6ba4 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -26,7 +26,7 @@ public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IE _datasetManager = datasetManager; } - public TrialResult Run(TrialSettings settings, IServiceProvider provider) + public TrialResult Run(TrialSettings settings) { var stopWatch = new Stopwatch(); stopWatch.Start(); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 7824ecf991..dde1ef06c4 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text; using System.Threading; @@ -59,9 +60,10 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async() { var context = new MLContext(1); - + var stopWatch = new Stopwatch(); + stopWatch.Start(); var experiment = context.Auto().CreateExperiment(); - experiment.SetTrainingTimeInSeconds(100) + experiment.SetTrainingTimeInSeconds(10) .SetTrialRunner((serviceProvider) => { var channel = serviceProvider.GetService(); @@ -79,8 +81,11 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled cts.CancelAfter(100); } }; - var res = await experiment.RunAsync(cts.Token); + + stopWatch.Stop(); + stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000); + cts.IsCancellationRequested.Should().BeTrue(); res.Metric.Should().BeGreaterThan(0); } @@ -278,11 +283,12 @@ public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings _logger = logger; } - public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) + public TrialResult Run(TrialSettings settings) { _logger.Info("Update Running Trial"); Task.Delay(_finishAfterNSeconds * 1000).Wait(_ct); _ct.ThrowIfCancellationRequested(); + _logger.Info("Update Completed Trial"); return new TrialResult { TrialSettings = settings, From 6de7519d69459d54bf2a4f7f9b4521f1a388b67b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 22 Aug 2022 12:38:04 -0700 Subject: [PATCH 18/25] clean up --- .../API/AutoMLExperimentExtension.cs | 12 --- .../API/SweepableExtension.cs | 41 +-------- .../SweepableEstimatorPipeline.cs | 2 +- ...IEstimatorAndMultiClassifiers.approved.txt | 85 +++++++++++++++++++ .../SweepableExtensionTest.cs | 32 ++----- 5 files changed, 99 insertions(+), 73 deletions(-) create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs index 5b43285b70..2598bc0189 100644 --- a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -12,7 +12,6 @@ namespace Microsoft.ML.AutoML.API { public static class AutoMLExperimentExtension { - public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView test) { var datasetManager = new TrainTestDatasetManager() @@ -89,17 +88,6 @@ public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, Swe return experiment; } - public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, SweepableEstimatorPipeline pipeline) - { - var res = new SweepablePipeline(); - foreach (var e in pipeline.Estimators) - { - res = res.Append(e); - } - - return experiment.SetPipeline(res); - } - private static AutoMLExperiment SetEvaluateMetric(this AutoMLExperiment experiment, TEvaluateMetricManager metricManager) where TEvaluateMetricManager : class, IEvaluateMetricManager { diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index e70f1c826b..506037d549 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -6,14 +6,9 @@ namespace Microsoft.ML.AutoML { public static class SweepableExtension { - public static SweepableEstimatorPipeline Append(this IEstimator estimator, SweepableEstimator estimator1) + public static SweepablePipeline Append(this IEstimator estimator, SweepableEstimator estimator1) { - return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); - } - - public static SweepableEstimatorPipeline Append(this SweepableEstimatorPipeline pipeline, IEstimator estimator1) - { - return pipeline.Append(new SweepableEstimator((context, parameter) => estimator1, new SearchSpace.SearchSpace())); + return new SweepablePipeline().Append(estimator).Append(estimator1); } public static SweepablePipeline Append(this SweepablePipeline pipeline, IEstimator estimator) @@ -21,33 +16,16 @@ public static SweepablePipeline Append(this SweepablePipeline pipeline, IEstimat return pipeline.Append(new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace())); } - public static SweepablePipeline Append(this SweepableEstimatorPipeline pipeline, SweepablePipeline sweepablePipeline) - { - var res = new SweepablePipeline(); - foreach (var estimator in pipeline.Estimators) - { - res = res.Append(estimator); - } - - return res.Append(sweepablePipeline); - } - - public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, SweepableEstimator estimator1) - { - return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); - } - public static SweepablePipeline Append(this SweepableEstimator estimator, SweepablePipeline estimator1) { return new SweepablePipeline().Append(estimator).Append(estimator1); } - public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, IEstimator estimator1) + public static SweepablePipeline Append(this SweepableEstimator estimator, IEstimator estimator1) { - return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); + return new SweepablePipeline().Append(estimator).Append(estimator1); } - public static SweepablePipeline Append(this IEstimator estimator, SweepablePipeline pipeline) { var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); @@ -71,16 +49,5 @@ public static SweepablePipeline Append(this IEstimator estimator, return pipeline; } - - public static SweepablePipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) - { - var res = new SweepablePipeline(); - foreach (var estimator in pipeline.Estimators) - { - res = res.Append(estimator); - } - - return res.Append(estimators); - } } } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs index f04b7ae63b..76e5682ba5 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(SweepableEstimatorPipelineConverter))] - public class SweepableEstimatorPipeline + internal class SweepableEstimatorPipeline { private readonly List _estimators; diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..b02699087d --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,85 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index afba4771b9..0afa1a1ea7 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -44,46 +44,32 @@ public SweepableExtensionTest(ITestOutputHelper output) } [Fact] - public void CreateSweepableEstimatorPipelineFromIEstimatorTest() + public void CreateSweepablePipelineFromIEstimatorTest() { var context = new MLContext(); var estimator = context.Transforms.Concatenate("output", "input"); var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); - pipeline.Should().BeOfType(); - pipeline.ToString().Should().Be("Unknown=>FastForestBinary"); - } - - [Fact] - public void AppendIEstimatorToSweepabeEstimatorPipelineTest() - { - var context = new MLContext(); - var estimator = context.Transforms.Concatenate("output", "input"); - var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); - pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); - - pipeline.Should().BeOfType(); - pipeline.ToString().Should().Be("Unknown=>FastForestBinary=>Unknown"); + pipeline.Should().BeOfType(); } [Fact] - public void CreateSweepableEstimatorPipelineFromSweepableEstimatorTest() + public void CreateSweepablePipelineFromSweepableEstimatorTest() { var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); var pipeline = estimator.Append(estimator); - pipeline.ToString().Should().Be("FastForestBinary=>FastForestBinary"); + pipeline.Should().BeOfType(); } [Fact] - public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorTest() + public void CreateSweepablePipelineFromSweepableEstimatorAndIEstimatorTest() { var context = new MLContext(); var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); var pipeline = estimator.Append(context.Transforms.Concatenate("output", "input")); - pipeline.ToString().Should().Be("FastForestBinary=>Unknown"); - + pipeline.Should().BeOfType(); } [Fact] @@ -102,7 +88,7 @@ public void CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers() + public void CreateSweepablePipelineFromIEstimatorAndMultiClassifiers() { var context = new MLContext(); SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") @@ -141,7 +127,7 @@ public void CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers() + public void CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers() { var context = new MLContext(); SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") @@ -155,7 +141,7 @@ public void CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassif [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray() + public void CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray() { var context = new MLContext(); SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") From ce582572f41c9a2355b686b1e63b36a5b61be656 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Aug 2022 11:36:16 -0700 Subject: [PATCH 19/25] fix comment and add comment in AutoMLExtension --- .../API/AutoMLExperimentExtension.cs | 68 +++++++++++++++++-- .../API/BinaryClassificationExperiment.cs | 14 ++-- .../API/MulticlassClassificationExperiment.cs | 14 ++-- .../AutoMLExperiment/IMetricManager.cs | 2 +- .../AutoMLExperiment/IMonitor.cs | 4 +- src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs | 2 +- 6 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs index 2598bc0189..7196583325 100644 --- a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -12,12 +12,20 @@ namespace Microsoft.ML.AutoML.API { public static class AutoMLExperimentExtension { - public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView test) + /// + /// Set train and validation dataset for . This will make uses + /// to train a model, and use to evaluate the model. + /// + /// + /// dataset for training a model. + /// dataset for validating a model during training. + /// + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView validation) { var datasetManager = new TrainTestDatasetManager() { TrainDataset = train, - TestDataset = test + TestDataset = validation }; experiment.ServiceCollection.AddSingleton(datasetManager); @@ -26,11 +34,26 @@ public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDat return experiment; } - public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, TrainTestData trainTestSplit) + /// + /// Set train and validation dataset for . This will make uses from + /// to train a model, and use from to evaluate the model. + /// + /// + /// a for train and validation. + /// + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, TrainTestData trainValidationSplit) { - return experiment.SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet); + return experiment.SetDataset(trainValidationSplit.TrainSet, trainValidationSplit.TestSet); } + /// + /// Set cross-validation dataset for . This will make use n= cross-validation split on + /// to train and evaluate a model. + /// + /// + /// dataset for cross-validation split. + /// + /// public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView dataset, int fold = 10) { var datasetManager = new CrossValidateDatasetManager() @@ -45,13 +68,30 @@ public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDat return experiment; } - + /// + /// Set as evaluation manager for . This will make + /// uses as evaluation metric. + /// + /// + /// evaluation metric. + /// label column. + /// predicted column. + /// public static AutoMLExperiment SetBinaryClassificationMetric(this AutoMLExperiment experiment, BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { - var metricManager = new BinaryMetricManager(metric, predictedColumn, labelColumn); + var metricManager = new BinaryMetricManager(metric, labelColumn, predictedColumn); return experiment.SetEvaluateMetric(metricManager); } + /// + /// Set as evaluation manager for . This will make + /// uses as evaluation metric. + /// + /// + /// evaluation metric. + /// label column. + /// predicted column. + /// public static AutoMLExperiment SetMulticlassClassificationMetric(this AutoMLExperiment experiment, MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { var metricManager = new MultiClassMetricManager() @@ -64,6 +104,15 @@ public static AutoMLExperiment SetMulticlassClassificationMetric(this AutoMLExpe return experiment.SetEvaluateMetric(metricManager); } + /// + /// Set as evaluation manager for . This will make + /// uses as evaluation metric. + /// + /// + /// evaluation metric. + /// label column. + /// score column. + /// public static AutoMLExperiment SetRegressionMetric(this AutoMLExperiment experiment, RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") { var metricManager = new RegressionMetricManager() @@ -76,6 +125,13 @@ public static AutoMLExperiment SetRegressionMetric(this AutoMLExperiment experim return experiment.SetEvaluateMetric(metricManager); } + /// + /// Set for training. This also make uses + /// , and for automl traininng as well. + /// + /// + /// + /// public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, SweepablePipeline pipeline) { experiment.AddSearchSpace(AutoMLExperiment.PipelineSearchspaceName, pipeline.SearchSpace); diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index e4ff36cc19..fe2f7317a4 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -150,7 +150,6 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - TrialResultMonitor monitor = null; _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); @@ -173,6 +172,9 @@ public override ExperimentResult Execute(IDataView } _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + + // set monitor + TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { var channel = provider.GetService(); @@ -199,12 +201,14 @@ public override ExperimentResult Execute(IDataView public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - TrialResultMonitor monitor = null; _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + + // set monitor + TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { var channel = provider.GetService(); @@ -252,12 +256,14 @@ public override ExperimentResult Execute(IDataView public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - TrialResultMonitor monitor = null; _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + + // set monitor + TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { var channel = provider.GetService(); @@ -414,7 +420,7 @@ public TrialResult Run(TrialSettings settings) } } - throw new ArgumentException("IMetricManager must be BinaryMetricManager and IDatasetManager must be either TrainTestSplitDatasetManager or CrossValidationDatasetManager"); + throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } } } diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 6305e43594..3e033a9ad6 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -141,7 +141,6 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - TrialResultMonitor monitor = null; _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); @@ -165,6 +164,9 @@ public override ExperimentResult Execute(IDataV _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + + // set monitor + TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { var channel = provider.GetService(); @@ -197,13 +199,15 @@ public override ExperimentResult Execute(IDataV public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - TrialResultMonitor monitor = null; _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + + // set monitor + TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { var channel = provider.GetService(); @@ -252,13 +256,15 @@ public override ExperimentResult Execute(IDataV public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - TrialResultMonitor monitor = null; _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); _experiment.SetPipeline(_pipeline); + + // set monitor + TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { var channel = provider.GetService(); @@ -422,7 +428,7 @@ public TrialResult Run(TrialSettings settings) } } - throw new ArgumentException("IMetricManager must be MultiMetricManager and IDatasetManager must be either TrainTestSplitDatasetManager or CrossValidationDatasetManager"); + throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index 8c4d6ab9bc..3c25268bfb 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -23,7 +23,7 @@ public interface IEvaluateMetricManager : IMetricManager internal class BinaryMetricManager : IEvaluateMetricManager { - public BinaryMetricManager(BinaryClassificationMetric metric, string predictedColumn, string labelColumn) + public BinaryMetricManager(BinaryClassificationMetric metric, string labelColumn, string predictedColumn) { Metric = metric; PredictedColumn = predictedColumn; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index a39f7d6d39..fb52ebe686 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -22,7 +22,9 @@ public interface IMonitor void ReportRunningTrial(TrialSettings setting); } - // this monitor redirects output result to context.log + /// + /// monitor that redirect output results to context.Log channel. + /// internal class MLContextMonitor : IMonitor { private readonly IChannel _logger; diff --git a/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs index 4b38926e64..1c2f8fb1e4 100644 --- a/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs @@ -10,7 +10,7 @@ namespace Microsoft.ML.AutoML { /// - /// propose hyper parameter using ECI index. + /// propose hyper parameter using ECI index and . /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer /// is more likely to be picked. /// From df379c57943518046b458169820b9a64a575a630 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Aug 2022 11:38:10 -0700 Subject: [PATCH 20/25] change namespace for AutoMLExtension --- src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs | 2 +- src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs | 1 - .../API/MulticlassClassificationExperiment.cs | 1 - test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 1 - test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs index 7196583325..f537f94f11 100644 --- a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -8,7 +8,7 @@ using Microsoft.Extensions.DependencyInjection; using static Microsoft.ML.DataOperationsCatalog; -namespace Microsoft.ML.AutoML.API +namespace Microsoft.ML.AutoML { public static class AutoMLExperimentExtension { diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index fe2f7317a4..b9e41062a3 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.Linq; using Microsoft.Extensions.DependencyInjection; -using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 3e033a9ad6..6d9517f8ff 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.Linq; using Microsoft.Extensions.DependencyInjection; -using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 8099059cad..ff2ba2474d 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading; using FluentAssertions; -using Microsoft.ML.AutoML.API; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index dde1ef06c4..f327b15b99 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -12,7 +12,6 @@ using FluentAssertions; using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; -using Microsoft.ML.AutoML.API; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Xunit; From 6be0a48cb41b33ccd1add79bacc70fa3d81cb2bf Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Aug 2022 11:42:40 -0700 Subject: [PATCH 21/25] fix build error --- docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs index d82fb2b44b..cda207ec09 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs @@ -3,7 +3,6 @@ using System.IO; using System.Linq; using System.Text; -using Microsoft.ML.AutoML.API; namespace Microsoft.ML.AutoML.Samples { From c4b51f64b21c4322f080702d9b4c7a978ef53b68 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Aug 2022 12:56:08 -0700 Subject: [PATCH 22/25] fix tests --- .../API/AutoMLExperimentExtension.cs | 4 +- .../AutoMLExperiment/AutoMLExperiment.cs | 4 - src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs | 4 +- ...lePipelineAndMultiClassifiers.approved.txt | 89 +++++++++++++++++++ ...ineAndSweepableEstimatorArray.approved.txt | 89 +++++++++++++++++++ 5 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs index f537f94f11..65c08bcf91 100644 --- a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -127,7 +127,7 @@ public static AutoMLExperiment SetRegressionMetric(this AutoMLExperiment experim /// /// Set for training. This also make uses - /// , and for automl traininng as well. + /// , and for automl traininng as well. /// /// /// @@ -139,7 +139,7 @@ public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, Swe experiment.SetTrialRunner(); experiment.SetMonitor(); - experiment.SetTuner(); + experiment.SetTuner(); return experiment; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 9e73e751b9..be3c4d3758 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -242,10 +242,6 @@ public async Task RunAsync(CancellationToken ct = default) // when error is fatal (like schema mismatch). throw; } - else - { - break; - } } } diff --git a/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs index 1c2f8fb1e4..641087b9b7 100644 --- a/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.AutoML /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer /// is more likely to be picked. /// - public class EciCfoTuner : ITuner + public class EciCostFrugalTuner : ITuner { private readonly Dictionary _tuners; private readonly PipelineProposer _pipelineProposer; @@ -22,7 +22,7 @@ public class EciCfoTuner : ITuner // the key is trial id, and value is the schema for that trial. private readonly IMetricManager _metricManager; - public EciCfoTuner(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) + public EciCostFrugalTuner(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) { _tuners = new Dictionary(); _pipelineProposer = new PipelineProposer(sweepablePipeline, settings, metricManager); diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..5ea3e5977e --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt @@ -0,0 +1,89 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..5ea3e5977e --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,89 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file From 6c201a893deacb8e83f1c48bfa87b5a63129c6b0 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Aug 2022 14:09:25 -0700 Subject: [PATCH 23/25] fix test --- .../AutoMLExperiment/AutoMLExperiment.cs | 11 +++++------ .../AutoMLExperimentTests.cs | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index be3c4d3758..e142579e14 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -91,7 +91,7 @@ public AutoMLExperiment AddSearchSpace(string key, SearchSpace.SearchSpace searc public AutoMLExperiment SetMonitor(TMonitor monitor) where TMonitor : class, IMonitor { - _serviceCollection.AddSingleton(ServiceDescriptor.Singleton(monitor)); + _serviceCollection.AddSingleton(monitor); return this; } @@ -99,7 +99,7 @@ public AutoMLExperiment SetMonitor(TMonitor monitor) public AutoMLExperiment SetMonitor() where TMonitor : class, IMonitor { - _serviceCollection.AddSingleton(ServiceDescriptor.Singleton()); + _serviceCollection.AddSingleton(); return this; } @@ -107,7 +107,7 @@ public AutoMLExperiment SetMonitor() public AutoMLExperiment SetMonitor(Func factory) where TMonitor : class, IMonitor { - _serviceCollection.AddSingleton(ServiceDescriptor.Singleton(factory)); + _serviceCollection.AddSingleton(factory); return this; } @@ -115,7 +115,7 @@ public AutoMLExperiment SetMonitor(Func fa public AutoMLExperiment SetTrialRunner(TTrialRunner runner) where TTrialRunner : class, ITrialRunner { - _serviceCollection.AddSingleton(ServiceDescriptor.Singleton(runner)); + _serviceCollection.AddSingleton(runner); return this; } @@ -209,8 +209,7 @@ public async Task RunAsync(CancellationToken ct = default) var parameter = tuner.Propose(setting); setting.Parameter = parameter; monitor?.ReportRunningTrial(setting); - var runner = serviceProvider.GetService(); - Contracts.Assert(runner != null, "trial runner can't be null"); + var runner = serviceProvider.GetRequiredService(); try { diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index f327b15b99..246ab5f66b 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -130,7 +130,7 @@ public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() experiment.SetDataset(context.Data.TrainTestSplit(data)) .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(10); + .SetTrainingTimeInSeconds(1); var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.8); From 6834273bb9ec966ea0a3578a33543a15e51dbd56 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Aug 2022 15:04:16 -0700 Subject: [PATCH 24/25] fix test --- .../API/MulticlassClassificationExperiment.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 6d9517f8ff..78c503673f 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -179,11 +179,6 @@ public override ExperimentResult Execute(IDataV return monitor; }); - monitor.OnTrialCompleted += (o, e) => - { - var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); - progressHandler?.Report(detail); - }; _experiment.SetTrialRunner(); _experiment.Run(); From 42811162a725a3b5af4cae8e1ddaa7a28f0387f3 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 24 Aug 2022 16:58:19 -0700 Subject: [PATCH 25/25] Update job-template.yml --- build/ci/job-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index 09cb464776..89f81e66eb 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -20,7 +20,7 @@ jobs: ${{ if eq(parameters.nightlyBuild, 'true') }}: timeoutInMinutes: 30 ${{ if and(eq(parameters.nightlyBuild, 'false'), eq(parameters.codeCoverage, 'false')) }}: - timeoutInMinutes: 90 + timeoutInMinutes: 120 ${{ if eq(parameters.codeCoverage, 'true') }}: timeoutInMinutes: 120 cancelTimeoutInMinutes: 10 @@ -239,4 +239,4 @@ jobs: displayName: Clean up runtime folder for package (Unix) - ${{ if eq(parameters.nightlyBuild, 'false') }}: - script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} $(testTargetFramework) - displayName: Build Packages \ No newline at end of file + displayName: Build Packages