From b668bda199950e95190a061e53bf8bc1a5279ce0 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 6 Jul 2022 12:52:54 -0700 Subject: [PATCH 1/6] reimplement binary experiment using AutoML experiment --- .../API/BinaryClassificationExperiment.cs | 232 ++++++++++++++++++ src/Microsoft.ML.AutoML/API/ExperimentBase.cs | 12 +- .../AutoMLExperiment/AutoMLExperiment.cs | 41 ++-- .../AutoMLExperiment/IMonitor.cs | 6 +- .../AutoMLExperiment/TrialResult.cs | 22 ++ .../AutoMLExperiment/TrialRunner.cs | 7 +- .../TrialSettingsProposer/PipelineProposer.cs | 10 +- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 55 ++++- 8 files changed, 342 insertions(+), 43 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 359c19fcb2..8e372528df 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -147,6 +147,9 @@ public enum BinaryClassificationTrainer /// public sealed class BinaryClassificationExperiment : ExperimentBase { + private readonly AutoMLExperiment _experiment; + private const string Features = "__Features__"; + internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings) : base(context, new BinaryMetricsAgent(context, settings.OptimizingMetric), @@ -155,6 +158,161 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti TaskKind.BinaryClassification, TrainerExtensionUtil.GetTrainerNames(settings.Trainers)) { + _experiment = context.Auto().CreateExperiment(); + } + + public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) + { + var label = columnInformation.LabelColumnName; + _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); + + // Cross val threshold for # of dataset rows -- + // If dataset has < threshold # of rows, use cross val. + // Else, run experiment using train-validate split. + const int crossValRowCountThreshold = 15000; + var rowCount = DatasetDimensionsUtil.CountRows(trainData, crossValRowCountThreshold); + // TODO + // split cross validation result according to sample key as well. + if (rowCount < crossValRowCountThreshold) + { + const int numCrossValFolds = 10; + _experiment.SetDataset(trainData, numCrossValFolds); + } + else + { + var splitData = Context.Data.TrainTestSplit(trainData); + _experiment.SetDataset(splitData.TrainSet, splitData.TestSet); + } + + MultiModelPipeline pipeline = new MultiModelPipeline(); + if (preFeaturizer != null) + { + pipeline = pipeline.Append(preFeaturizer); + } + + pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) + .Append(Context.Auto().BinaryClassification(label, Features)); + _experiment.SetPipeline(pipeline); + + var monitor = new BinaryClassificationTrialResultMonitor(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = ToRunDetail(e); + progressHandler?.Report(detail); + }; + + _experiment.SetMonitor(monitor); + _experiment.Run(); + + var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e)); + var bestRun = ToRunDetail(monitor.BestRun); + var result = new ExperimentResult(runDetails, bestRun); + + return result; + } + + public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) + { + var label = columnInformation.LabelColumnName; + _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); + _experiment.SetDataset(trainData, validationData); + + MultiModelPipeline pipeline = new MultiModelPipeline(); + if (preFeaturizer != null) + { + pipeline = pipeline.Append(preFeaturizer); + } + + pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__")) + .Append(Context.Auto().BinaryClassification(label, featureColumnName: Features)); + + _experiment.SetPipeline(pipeline); + var monitor = new BinaryClassificationTrialResultMonitor(); + monitor.OnTrialCompleted += (o, e) => + { + var detail = ToRunDetail(e); + progressHandler?.Report(detail); + }; + + _experiment.SetMonitor(monitor); + _experiment.Run(); + + var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e)); + var bestRun = ToRunDetail(monitor.BestRun); + var result = new ExperimentResult(runDetails, bestRun); + + return result; + } + + public override ExperimentResult Execute(IDataView trainData, IDataView validationData, string labelColumnName = "Label", IEstimator preFeaturizer = null, IProgress> progressHandler = null) + { + var columnInformation = new ColumnInformation() + { + LabelColumnName = labelColumnName, + }; + + return this.Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); + } + + public override ExperimentResult Execute(IDataView trainData, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) + { + var columnInformation = new ColumnInformation() + { + LabelColumnName = labelColumnName, + SamplingKeyColumnName = samplingKeyColumn, + }; + + return this.Execute(trainData, columnInformation, preFeaturizer, progressHandler); + } + + public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) + { + var label = columnInformation.LabelColumnName; + _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); + _experiment.SetDataset(trainData, (int)numberOfCVFolds); + + MultiModelPipeline pipeline = new MultiModelPipeline(); + if (preFeaturizer != null) + { + pipeline = pipeline.Append(preFeaturizer); + } + + pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__")) + .Append(Context.Auto().BinaryClassification(label, featureColumnName: Features)); + + _experiment.SetPipeline(pipeline); + + var monitor = new BinaryClassificationTrialResultMonitor(); + monitor.OnTrialCompleted += (o, e) => + { + var runDetails = ToCrossValidationRunDetail(e); + + progressHandler?.Report(runDetails); + }; + + _experiment.SetMonitor(monitor); + _experiment.Run(); + + var runDetails = monitor.RunDetails.Select(e => ToCrossValidationRunDetail(e)); + var bestResult = ToCrossValidationRunDetail(monitor.BestRun); + + var result = new CrossValidationExperimentResult(runDetails, bestResult); + + return result; + } + + public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) + { + var columnInformation = new ColumnInformation() + { + LabelColumnName = labelColumnName, + SamplingKeyColumnName = samplingKeyColumn, + }; + + return this.Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); } private protected override RunDetail GetBestRun(IEnumerable> results) @@ -166,5 +324,79 @@ private protected override CrossValidationRunDetail { return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing); } + + private RunDetail ToRunDetail(BinaryClassificationTrialResult result) + { + var pipeline = result.TrialSettings.Pipeline; + var trainerName = pipeline.ToString(); + var parameter = result.TrialSettings.Parameter; + var estimator = pipeline.BuildTrainingPipeline(Context, parameter); + var modelContainer = new ModelContainer(Context, result.Model); + return new RunDetail(trainerName, estimator, null, modelContainer, result.BinaryClassificationMetrics, result.Exception); + } + + private CrossValidationRunDetail ToCrossValidationRunDetail(BinaryClassificationTrialResult result) + { + var pipeline = result.TrialSettings.Pipeline; + var trainerName = pipeline.ToString(); + var parameter = result.TrialSettings.Parameter; + var estimator = pipeline.BuildTrainingPipeline(Context, parameter); + var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult(new ModelContainer(Context, m.Model), m.Metrics, result.Exception)); + return new CrossValidationRunDetail(trainerName, estimator, null, crossValidationResult); + } + } + + internal class BinaryClassificationTrialResultMonitor : IMonitor + { + public BinaryClassificationTrialResultMonitor() + { + this.RunDetails = new List(); + } + + public event EventHandler OnTrialCompleted; + + public List RunDetails { get; } + + public BinaryClassificationTrialResult BestRun { get; private set; } + + public void ReportBestTrial(TrialResult result) + { + if (result is BinaryClassificationTrialResult binaryClassificationResult) + { + BestRun = binaryClassificationResult; + } + else + { + throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}"); + } + } + + public void ReportCompletedTrial(TrialResult result) + { + if (result is BinaryClassificationTrialResult binaryClassificationResult) + { + RunDetails.Add(binaryClassificationResult); + OnTrialCompleted?.Invoke(this, binaryClassificationResult); + } + else + { + throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}"); + } + } + + public void ReportFailTrial(TrialSettings settings, Exception exp) + { + var result = new BinaryClassificationTrialResult + { + TrialSettings = settings, + Exception = exp, + }; + + RunDetails.Add(result); + } + + public void ReportRunningTrial(TrialSettings setting) + { + } } } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentBase.cs b/src/Microsoft.ML.AutoML/API/ExperimentBase.cs index eb6db27cdb..dd2f0ed7e7 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentBase.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentBase.cs @@ -64,7 +64,7 @@ internal ExperimentBase(MLContext context, /// /// Depending on the size of your data, the AutoML experiment could take a long time to execute. /// - public ExperimentResult Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label, + public virtual ExperimentResult Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label, string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { ColumnInformation columnInformation; @@ -106,7 +106,7 @@ public ExperimentResult Execute(IDataView trainData, string labelColum /// /// Depending on the size of your data, the AutoML experiment could take a long time to execute. /// - public ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, + public virtual ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { // Cross val threshold for # of dataset rows -- @@ -156,7 +156,7 @@ private string GetSamplingKey(string groupIdColumnName, string samplingKeyColumn /// /// Depending on the size of your data, the AutoML experiment could take a long time to execute. /// - public ExperimentResult Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator preFeaturizer = null, IProgress> progressHandler = null) + public virtual ExperimentResult Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var columnInformation = (_task == TaskKind.Ranking) ? new ColumnInformation() { LabelColumnName = labelColumnName, GroupIdColumnName = DefaultColumnNames.GroupId } : @@ -184,7 +184,7 @@ public ExperimentResult Execute(IDataView trainData, IDataView validat /// /// Depending on the size of your data, the AutoML experiment could take a long time to execute. /// - public ExperimentResult Execute(IDataView trainData, IDataView validationData, + public virtual ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { @@ -214,7 +214,7 @@ public ExperimentResult Execute(IDataView trainData, IDataView validat /// /// Depending on the size of your data, the AutoML experiment could take a long time to execute. /// - public CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, + public virtual CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { @@ -244,7 +244,7 @@ public CrossValidationExperimentResult Execute(IDataView trainData, ui /// /// Depending on the size of your data, the AutoML experiment could take a long time to execute. /// - public CrossValidationExperimentResult Execute(IDataView trainData, + public virtual CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label, string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index eef54851c3..c179e95907 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -155,7 +155,7 @@ public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) return this; } - public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") + public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { var metricManager = new BinaryMetricManager() { @@ -169,7 +169,7 @@ public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, str return this; } - public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") + public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") { var metricManager = new MultiClassMetricManager() { @@ -231,23 +231,23 @@ public async Task RunAsync(CancellationToken ct = default) while (true) { - try + if (cts.Token.IsCancellationRequested) { - if (cts.Token.IsCancellationRequested) - { - break; - } + break; + } + var setting = new TrialSettings() + { + ExperimentSettings = _settings, + TrialId = trialNum++, + }; - var setting = new TrialSettings() - { - ExperimentSettings = _settings, - TrialId = trialNum++, - }; - - setting = pipelineProposer.Propose(setting); - setting = hyperParameterProposer.Propose(setting); - monitor.ReportRunningTrial(setting); - var runner = runnerFactory.CreateTrialRunner(); + setting = pipelineProposer.Propose(setting); + setting = hyperParameterProposer.Propose(setting); + monitor.ReportRunningTrial(setting); + var runner = runnerFactory.CreateTrialRunner(); + + try + { var trialResult = runner.Run(setting, serviceProvider); monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); @@ -261,7 +261,7 @@ public async Task RunAsync(CancellationToken ct = default) monitor.ReportBestTrial(trialResult); } } - catch (Exception) + catch (Exception ex) { if (cts.Token.IsCancellationRequested) { @@ -269,6 +269,11 @@ public async Task RunAsync(CancellationToken ct = default) } else { + // TODO + // it's questionable on whether to abort the entire training process + // for a single fail trial. We should make it an option and only exit + // when error is fatal (like schema mismatch). + monitor.ReportFailTrial(setting, ex); throw; } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 1d17908370..cc9cbe0035 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -17,7 +17,7 @@ public interface IMonitor void ReportBestTrial(TrialResult result); - void ReportFailTrial(TrialResult result); + void ReportFailTrial(TrialSettings settings, Exception exception = null); void ReportRunningTrial(TrialSettings setting); } @@ -49,9 +49,9 @@ public void ReportCompletedTrial(TrialResult result) _completedTrials.Add(result); } - public void ReportFailTrial(TrialResult result) + public void ReportFailTrial(TrialSettings settings, Exception exception = null) { - _logger.Info($"Update Failed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + _logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {settings.Pipeline}"); } public void ReportRunningTrial(TrialSettings setting) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index bceaa08cde..0b8625f37e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -2,6 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; +using System.Collections; +using System.Collections.Generic; +using Microsoft.ML.Data; +using static Microsoft.ML.TrainCatalogBase; + namespace Microsoft.ML.AutoML { public class TrialResult @@ -14,4 +20,20 @@ public class TrialResult public double DurationInMilliseconds { get; set; } } + + /// + /// TrialResult with Binary Classification Metrics + /// + internal class BinaryClassificationTrialResult : TrialResult + { + public BinaryClassificationMetrics BinaryClassificationMetrics { get; set; } + + public IEnumerable> CrossValidationMetrics { get; set; } + + public Exception Exception { get; set; } + + public bool IsSucceed { get => Exception == null; } + + public bool IsCrossValidation { get => CrossValidationMetrics == null; } + } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index a0e8cb3e5f..d765599267 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -56,12 +56,14 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) stopWatch.Stop(); - return new TrialResult() + return new BinaryClassificationTrialResult() { Metric = metric, Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + BinaryClassificationMetrics = res.Metrics, + CrossValidationMetrics = metrics, }; } @@ -108,12 +110,13 @@ public TrialResult Run(TrialSettings settings, IServiceProvider provider) stopWatch.Stop(); - return new TrialResult() + return new BinaryClassificationTrialResult() { Metric = metric, Model = model, TrialSettings = settings, DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + BinaryClassificationMetrics = metrics, }; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index d40e6d6b6a..30134c0da0 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -53,15 +53,15 @@ public PipelineProposer(AutoMLExperimentSettings settings) { EstimatorType.LbfgsPoissonRegressionRegression, 4.73 }, { EstimatorType.FastForestOva, 4.283 }, { EstimatorType.FastTreeOva, 3.701 }, - { EstimatorType.LightGbmMulti, 14.765 }, - { EstimatorType.SdcaMaximumEntropyMulti, 1.129 }, - { EstimatorType.SdcaLogisticRegressionOva, 3.16 }, + { EstimatorType.LightGbmMulti, 4.765 }, + { EstimatorType.SdcaMaximumEntropyMulti, 10.129 }, + { EstimatorType.SdcaLogisticRegressionOva, 13.16 }, { EstimatorType.LbfgsMaximumEntropyMulti, 7.980 }, { EstimatorType.LbfgsLogisticRegressionOva, 11.513 }, - { EstimatorType.LightGbmBinary, 14.765 }, + { EstimatorType.LightGbmBinary, 4.765 }, { EstimatorType.FastTreeBinary, 3.701 }, { EstimatorType.FastForestBinary, 4.283 }, - { EstimatorType.SdcaLogisticRegressionBinary, 3.16 }, + { EstimatorType.SdcaLogisticRegressionBinary, 13.16 }, { EstimatorType.LbfgsLogisticRegressionBinary, 11.513 }, { EstimatorType.ForecastBySsa, 1 }, { EstimatorType.ImageClassificationMulti, 1 }, diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index c914d522b5..3709f32d55 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -56,7 +56,7 @@ public void AutoFitContextLogTest() } [Fact] - public void AutoFitBinaryTest() + public void AutoFit_UCI_Adult_Test() { var context = new MLContext(1); var dataPath = DatasetUtil.GetUciAdultDataset(); @@ -64,7 +64,7 @@ public void AutoFitBinaryTest() var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); var result = context.Auto() - .CreateBinaryClassificationExperiment(0) + .CreateBinaryClassificationExperiment(10) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -72,6 +72,40 @@ public void AutoFitBinaryTest() Assert.NotNull(result.BestRun.TrainerName); } + [Fact] + public void AutoFit_UCI_Adult_Train_Test_Split_Test() + { + var context = new MLContext(1); + var dataPath = DatasetUtil.GetUciAdultDataset(); + var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); + var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); + var trainData = textLoader.Load(dataPath); + var dataTrainTest = context.Data.TrainTestSplit(trainData); + var result = context.Auto() + .CreateBinaryClassificationExperiment(10) + .Execute(dataTrainTest.TrainSet, dataTrainTest.TestSet, DatasetUtil.UciAdultLabel); + Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); + Assert.NotNull(result.BestRun.Estimator); + Assert.NotNull(result.BestRun.Model); + Assert.NotNull(result.BestRun.TrainerName); + } + + [Fact] + public void AutoFit_UCI_Adult_CrossValidation_10_Test() + { + var context = new MLContext(1); + var dataPath = DatasetUtil.GetUciAdultDataset(); + var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); + var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); + var trainData = textLoader.Load(dataPath); + var result = context.Auto() + .CreateBinaryClassificationExperiment(10) + .Execute(trainData, 10, DatasetUtil.UciAdultLabel); + Assert.True(result.BestRun.Results.Select(x => x.ValidationMetrics.Accuracy).Min() > 0.70); + Assert.NotNull(result.BestRun.Estimator); + Assert.NotNull(result.BestRun.TrainerName); + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -401,21 +435,25 @@ public void AutoFitWithPresplittedData() var dataCV = context.Data.CrossValidationSplit(dataFull, numberOfFolds: 2); var modelFull = context.Auto() - .CreateBinaryClassificationExperiment(0) + .CreateBinaryClassificationExperiment(10) .Execute(dataFull, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun .Model; + // AutoMLExperiment can't run on canceled context. + // Therefore, we need to create a new context after an experiment is completed. + context = new MLContext(1); var modelTrainTest = context.Auto() - .CreateBinaryClassificationExperiment(0) + .CreateBinaryClassificationExperiment(10) .Execute(dataTrainTest.TrainSet, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun .Model; + context = new MLContext(1); var modelCV = context.Auto() - .CreateBinaryClassificationExperiment(0) + .CreateBinaryClassificationExperiment(10) .Execute(dataCV.First().TrainSet, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun @@ -429,9 +467,9 @@ public void AutoFitWithPresplittedData() var resTrainTest = model.Transform(dataTrainTest.TrainSet); var resCV = model.Transform(dataCV.First().TrainSet); - Assert.Equal(30, resFull.Schema.Count); - Assert.Equal(30, resTrainTest.Schema.Count); - Assert.Equal(30, resCV.Schema.Count); + Assert.Equal(31, resFull.Schema.Count); + Assert.Equal(31, resTrainTest.Schema.Count); + Assert.Equal(31, resCV.Schema.Count); foreach (var col in resFull.Schema) { @@ -439,7 +477,6 @@ public void AutoFitWithPresplittedData() Assert.Equal(col.Name, resCV.Schema[col.Index].Name); } } - } [LightGBMFact] From 7964bbd658b440d811e042e0b06f2f8611d12a34 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 6 Jul 2022 12:54:07 -0700 Subject: [PATCH 2/6] fix build error --- src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index 6e401d87d7..6aea99367d 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -55,7 +55,7 @@ public void ReportCompletedTrial(TrialResult result) ThrottledUpdate(); } - public void ReportFailTrial(TrialResult result) + public void ReportFailTrial(TrialSettings setting, Exception exp = null) { // TODO figure out what to do with failed trials. ThrottledUpdate(); From dfc76a96c4bc18a8501027afcb983f9acc03fbfa Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 11 Jul 2022 11:50:21 -0700 Subject: [PATCH 3/6] fix tests --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 3709f32d55..6894ff58b6 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -55,7 +55,7 @@ public void AutoFitContextLogTest() "were not relayed to the main MLContext."); } - [Fact] + [LightGBMFact] public void AutoFit_UCI_Adult_Test() { var context = new MLContext(1); @@ -72,7 +72,7 @@ public void AutoFit_UCI_Adult_Test() Assert.NotNull(result.BestRun.TrainerName); } - [Fact] + [LightGBMFact] public void AutoFit_UCI_Adult_Train_Test_Split_Test() { var context = new MLContext(1); @@ -90,7 +90,7 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() Assert.NotNull(result.BestRun.TrainerName); } - [Fact] + [LightGBMFact] public void AutoFit_UCI_Adult_CrossValidation_10_Test() { var context = new MLContext(1); From 8446e6e2f0d681eeefac98974b11193c0eeb0e4b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 13 Jul 2022 16:23:59 -0700 Subject: [PATCH 4/6] force canceling when there's complete trial --- .../AutoMLExperiment/AutoMLExperiment.cs | 8 +++++++- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index c179e95907..afc8dba450 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -219,7 +219,13 @@ public async Task RunAsync(CancellationToken ct = default) _settings.CancellationToken = ct; cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); _settings.CancellationToken.Register(() => cts.Cancel()); - cts.Token.Register(() => _context.CancelExecution()); + cts.Token.Register(() => + { + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + _context.CancelExecution(); + }); InitializeServiceCollection(); var serviceProvider = _serviceCollection.BuildServiceProvider(); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 6894ff58b6..3063f29dc8 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -64,7 +64,7 @@ public void AutoFit_UCI_Adult_Test() var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); var result = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(1) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -82,7 +82,7 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() var trainData = textLoader.Load(dataPath); var dataTrainTest = context.Data.TrainTestSplit(trainData); var result = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(1) .Execute(dataTrainTest.TrainSet, dataTrainTest.TestSet, DatasetUtil.UciAdultLabel); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -99,7 +99,7 @@ public void AutoFit_UCI_Adult_CrossValidation_10_Test() var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); var result = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(1) .Execute(trainData, 10, DatasetUtil.UciAdultLabel); Assert.True(result.BestRun.Results.Select(x => x.ValidationMetrics.Accuracy).Min() > 0.70); Assert.NotNull(result.BestRun.Estimator); From c10800fe70e2191186b00565308b6764b2fd0a51 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Fri, 15 Jul 2022 11:33:28 -0700 Subject: [PATCH 5/6] Update AutoFitTests.cs --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 3063f29dc8..d57d181a6c 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -419,7 +419,7 @@ public void AutoFitRecommendationTest() } } - [Fact] + [LightGbmFact] public void AutoFitWithPresplittedData() { // Models created in AutoML should work over the same data, From ae4148c5405d8d68669212cf0a0d8442c0a953aa Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Fri, 15 Jul 2022 11:50:02 -0700 Subject: [PATCH 6/6] Update AutoFitTests.cs --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index d57d181a6c..6860391591 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -419,7 +419,7 @@ public void AutoFitRecommendationTest() } } - [LightGbmFact] + [LightGBMFact] public void AutoFitWithPresplittedData() { // Models created in AutoML should work over the same data,