diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index 09cb464776..89f81e66eb 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -20,7 +20,7 @@ jobs: ${{ if eq(parameters.nightlyBuild, 'true') }}: timeoutInMinutes: 30 ${{ if and(eq(parameters.nightlyBuild, 'false'), eq(parameters.codeCoverage, 'false')) }}: - timeoutInMinutes: 90 + timeoutInMinutes: 120 ${{ if eq(parameters.codeCoverage, 'true') }}: timeoutInMinutes: 120 cancelTimeoutInMinutes: 10 @@ -239,4 +239,4 @@ jobs: displayName: Clean up runtime folder for package (Unix) - ${{ if eq(parameters.nightlyBuild, 'false') }}: - script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} $(testTargetFramework) - displayName: Build Packages \ No newline at end of file + displayName: Build Packages diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs index 2f4254ab8a..cda207ec09 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Cifar10.cs @@ -40,7 +40,7 @@ public static void Run() experiment.SetDataset(trainDataset, testDataset) .SetPipeline(pipeline) - .SetEvaluateMetric(MulticlassClassificationMetric.MicroAccuracy) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MicroAccuracy) .SetTrainingTimeInSeconds(200); var result = experiment.Run(); diff --git a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs index 7868a24b4c..8d06caad80 100644 --- a/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs +++ b/src/Microsoft.ML.AutoML.Interactive/AutoMLMonitorKernelExtension.cs @@ -51,7 +51,7 @@ private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) var bestTrialParam = JsonSerializer.Serialize(monitor.BestTrial.TrialSettings.Parameter, new JsonSerializerOptions() { WriteIndented = true, }); summary.Add(h3("Best Trial")); summary.Add(p($"Id: {monitor.BestTrial.TrialSettings.TrialId}")); - summary.Add(p($"Trainer: {monitor.BestTrial.TrialSettings.Pipeline}".Replace("Unknown=>", ""))); + summary.Add(p($"Trainer: {monitor.SweepablePipeline.ToString(monitor.BestTrial.TrialSettings.Parameter)}".Replace("Unknown=>", ""))); summary.Add(p($"Parameters: {bestTrialParam}")); } if (monitor.ActiveTrial != null) @@ -61,7 +61,7 @@ private static void WriteSummary(NotebookMonitor monitor, TextWriter writer) summary.Add(h3("Active Trial")); summary.Add(p($"Id: {monitor.ActiveTrial.TrialId}")); - summary.Add(p($"Trainer: {monitor.ActiveTrial.Pipeline}".Replace("Unknown=>", ""))); + summary.Add(p($"Trainer: {monitor.SweepablePipeline.ToString(monitor.ActiveTrial.Parameter)}".Replace("Unknown=>", ""))); summary.Add(p($"Parameters: {activeTrialParam}")); } diff --git a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs index 6aea99367d..81b22a9435 100644 --- a/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs +++ b/src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs @@ -17,17 +17,19 @@ public class NotebookMonitor : IMonitor private readonly ActionThrottler _updateThrottler; private DisplayedValue _valueToUpdate; + public SweepablePipeline SweepablePipeline { get; private set; } public TrialResult BestTrial { get; set; } public TrialResult MostRecentTrial { get; set; } public TrialSettings ActiveTrial { get; set; } public List CompletedTrials { get; set; } public DataFrame TrialData { get; set; } - public NotebookMonitor() + public NotebookMonitor(SweepablePipeline pipeline) { CompletedTrials = new List(); TrialData = new DataFrame(new PrimitiveDataFrameColumn("Trial"), new PrimitiveDataFrameColumn("Metric"), new StringDataFrameColumn("Trainer"), new StringDataFrameColumn("Parameters")); _updateThrottler = new ActionThrottler(Update, TimeSpan.FromSeconds(5)); + SweepablePipeline = pipeline; } public void ReportBestTrial(TrialResult result) @@ -48,7 +50,7 @@ public void ReportCompletedTrial(TrialResult result) { new KeyValuePair("Trial",result.TrialSettings.TrialId), new KeyValuePair("Metric", result.Metric), - new KeyValuePair("Trainer",result.TrialSettings.Pipeline.ToString().Replace("Unknown=>","")), + new KeyValuePair("Trainer",SweepablePipeline.ToString(result.TrialSettings.Parameter).Replace("Unknown=>","")), new KeyValuePair("Parameters",activeRunParam), }, true); diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index 441a416cc0..ed8a2a9ef4 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -326,7 +326,7 @@ public AutoMLExperiment CreateExperiment(AutoMLExperiment.AutoMLExperimentSettin /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. /// if provided, use it as search space for sdca, otherwise the default search space will be used. /// - public SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + public SweepablePipeline BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -377,7 +377,7 @@ public SweepableEstimator[] BinaryClassification(string labelColumnName = Defaul res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } - return res.ToArray(); + return new SweepablePipeline().Append(res.ToArray()); } /// @@ -402,7 +402,7 @@ public SweepableEstimator[] BinaryClassification(string labelColumnName = Defaul /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. /// if provided, use it as search space for sdca, otherwise the default search space will be used. /// - public SweepableEstimator[] MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + public SweepablePipeline MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -455,7 +455,7 @@ public SweepableEstimator[] MultiClassification(string labelColumnName = Default res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } - return res.ToArray(); + return new SweepablePipeline().Append(res.ToArray()); } /// @@ -480,7 +480,7 @@ public SweepableEstimator[] MultiClassification(string labelColumnName = Default /// if provided, use it as search space for lbfgs, otherwise the default search space will be used. /// if provided, use it as search space for sdca, otherwise the default search space will be used. /// - public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + public SweepablePipeline Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) { @@ -531,7 +531,7 @@ public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNam res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } - return res.ToArray(); + return new SweepablePipeline().Append(res.ToArray()); } /// @@ -539,7 +539,7 @@ public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNam /// /// output column name. /// input column name. - internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inputColumnName) + internal SweepablePipeline TextFeaturizer(string outputColumnName, string inputColumnName) { var option = new FeaturizeTextOption { @@ -547,7 +547,7 @@ internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inp OutputColumnName = outputColumnName, }; - return new[] { SweepableEstimatorFactory.CreateFeaturizeText(option) }; + return new SweepablePipeline().Append(new[] { SweepableEstimatorFactory.CreateFeaturizeText(option) }); } /// @@ -555,7 +555,7 @@ internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inp /// /// output column names. /// input column names. - internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, string[] inputColumnNames) + internal SweepablePipeline NumericFeaturizer(string[] outputColumnNames, string[] inputColumnNames) { Contracts.CheckValue(inputColumnNames, nameof(inputColumnNames)); Contracts.CheckValue(outputColumnNames, nameof(outputColumnNames)); @@ -566,7 +566,7 @@ internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, stri OutputColumnNames = outputColumnNames, }; - return new[] { SweepableEstimatorFactory.CreateReplaceMissingValues(replaceMissingValueOption) }; + return new SweepablePipeline().Append(new[] { SweepableEstimatorFactory.CreateReplaceMissingValues(replaceMissingValueOption) }); } /// @@ -597,7 +597,7 @@ internal SweepableEstimator[] BooleanFeaturizer(string[] outputColumnNames, stri /// /// output column names. /// input column names. - internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, string[] inputColumnNames) + internal SweepablePipeline CatalogFeaturizer(string[] outputColumnNames, string[] inputColumnNames) { Contracts.Check(outputColumnNames.Count() == inputColumnNames.Count() && outputColumnNames.Count() > 0, "outputColumnNames and inputColumnNames must have the same length and greater than 0"); @@ -607,10 +607,10 @@ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, stri OutputColumnNames = outputColumnNames, }; - return new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) }; + return new SweepablePipeline().Append(new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) }); } - internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string inputColumnName) + internal SweepablePipeline ImagePathFeaturizer(string outputColumnName, string inputColumnName) { // load image => resize image (224, 224) => extract pixels => dnn featurizer var loadImageOption = new LoadImageOption @@ -640,7 +640,7 @@ internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string OutputColumnName = outputColumnName, }; - var pipeline = new MultiModelPipeline(); + var pipeline = new SweepablePipeline(); return pipeline.Append(SweepableEstimatorFactory.CreateLoadImages(loadImageOption)) .Append(SweepableEstimatorFactory.CreateResizeImages(resizeImageOption)) @@ -660,7 +660,7 @@ internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string /// columns that should be treated as image path. If not specified, it will automatically infer if a column is catalog or not. /// output feature column. /// columns that won't be included when featurizing, like label - public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) + public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) { Contracts.CheckValue(data, nameof(data)); @@ -727,16 +727,18 @@ public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = " /// input data. /// column information. /// output feature column. - /// A for featurization. - public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features") + /// A for featurization. + public SweepablePipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features") { Contracts.CheckValue(data, nameof(data)); Contracts.CheckValue(columnInformation, nameof(columnInformation)); var columnPurposes = PurposeInference.InferPurposes(this._context, data, columnInformation); var textFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.TextFeature); - var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature && data.Schema[c.ColumnIndex].Type != BooleanDataViewType.Instance); - var booleanFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature && data.Schema[c.ColumnIndex].Type == BooleanDataViewType.Instance); + var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature + && data.Schema[c.ColumnIndex].Type != BooleanDataViewType.Instance + && !(data.Schema[c.ColumnIndex].Type is VectorDataViewType vt && vt.ItemType == BooleanDataViewType.Instance)).ToArray(); + var booleanFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature && !numericFeatures.Contains(c)); var catalogFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.CategoricalFeature); var imagePathFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.ImagePath); var textFeatureColumnNames = textFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray(); @@ -745,7 +747,7 @@ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInf var imagePathColumnNames = imagePathFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray(); var booleanFeatureColumnNames = booleanFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray(); - var pipeline = new MultiModelPipeline(); + var pipeline = new SweepablePipeline(); if (numericFeatureColumnNames.Length > 0) { pipeline = pipeline.Append(this.NumericFeaturizer(numericFeatureColumnNames, numericFeatureColumnNames)); diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs new file mode 100644 index 0000000000..65c08bcf91 --- /dev/null +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using static Microsoft.ML.DataOperationsCatalog; + +namespace Microsoft.ML.AutoML +{ + public static class AutoMLExperimentExtension + { + /// + /// Set train and validation dataset for . This will make uses + /// to train a model, and use to evaluate the model. + /// + /// + /// dataset for training a model. + /// dataset for validating a model during training. + /// + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView train, IDataView validation) + { + var datasetManager = new TrainTestDatasetManager() + { + TrainDataset = train, + TestDataset = validation + }; + + experiment.ServiceCollection.AddSingleton(datasetManager); + experiment.ServiceCollection.AddSingleton(datasetManager); + + return experiment; + } + + /// + /// Set train and validation dataset for . This will make uses from + /// to train a model, and use from to evaluate the model. + /// + /// + /// a for train and validation. + /// + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, TrainTestData trainValidationSplit) + { + return experiment.SetDataset(trainValidationSplit.TrainSet, trainValidationSplit.TestSet); + } + + /// + /// Set cross-validation dataset for . This will make use n= cross-validation split on + /// to train and evaluate a model. + /// + /// + /// dataset for cross-validation split. + /// + /// + public static AutoMLExperiment SetDataset(this AutoMLExperiment experiment, IDataView dataset, int fold = 10) + { + var datasetManager = new CrossValidateDatasetManager() + { + Dataset = dataset, + Fold = fold, + }; + + experiment.ServiceCollection.AddSingleton(datasetManager); + experiment.ServiceCollection.AddSingleton(datasetManager); + + return experiment; + } + + /// + /// Set as evaluation manager for . This will make + /// uses as evaluation metric. + /// + /// + /// evaluation metric. + /// label column. + /// predicted column. + /// + public static AutoMLExperiment SetBinaryClassificationMetric(this AutoMLExperiment experiment, BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + { + var metricManager = new BinaryMetricManager(metric, labelColumn, predictedColumn); + return experiment.SetEvaluateMetric(metricManager); + } + + /// + /// Set as evaluation manager for . This will make + /// uses as evaluation metric. + /// + /// + /// evaluation metric. + /// label column. + /// predicted column. + /// + public static AutoMLExperiment SetMulticlassClassificationMetric(this AutoMLExperiment experiment, MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + { + var metricManager = new MultiClassMetricManager() + { + Metric = metric, + PredictedColumn = predictedColumn, + LabelColumn = labelColumn, + }; + + return experiment.SetEvaluateMetric(metricManager); + } + + /// + /// Set as evaluation manager for . This will make + /// uses as evaluation metric. + /// + /// + /// evaluation metric. + /// label column. + /// score column. + /// + public static AutoMLExperiment SetRegressionMetric(this AutoMLExperiment experiment, RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") + { + var metricManager = new RegressionMetricManager() + { + Metric = metric, + ScoreColumn = scoreColumn, + LabelColumn = labelColumn, + }; + + return experiment.SetEvaluateMetric(metricManager); + } + + /// + /// Set for training. This also make uses + /// , and for automl traininng as well. + /// + /// + /// + /// + public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, SweepablePipeline pipeline) + { + experiment.AddSearchSpace(AutoMLExperiment.PipelineSearchspaceName, pipeline.SearchSpace); + experiment.ServiceCollection.AddSingleton(pipeline); + + experiment.SetTrialRunner(); + experiment.SetMonitor(); + experiment.SetTuner(); + + return experiment; + } + + private static AutoMLExperiment SetEvaluateMetric(this AutoMLExperiment experiment, TEvaluateMetricManager metricManager) + where TEvaluateMetricManager : class, IEvaluateMetricManager + { + experiment.ServiceCollection.AddSingleton(metricManager); + experiment.ServiceCollection.AddSingleton(metricManager); + experiment.SetIsMaximize(metricManager.IsMaximize); + + return experiment; + } + } +} diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 99c7393ed3..b9e41062a3 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -4,11 +4,15 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; +using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; +using static Microsoft.ML.TrainCatalogBase; namespace Microsoft.ML.AutoML { @@ -129,6 +133,7 @@ public sealed class BinaryClassificationExperiment : ExperimentBase Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); // Cross val threshold for # of dataset rows -- @@ -164,22 +169,29 @@ public override ExperimentResult Execute(IDataView var splitData = Context.Data.TrainTestSplit(trainData); _experiment.SetDataset(splitData.TrainSet, splitData.TestSet); } + _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var pipeline = this.CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); - - var monitor = new TrialResultMonitor(Context); - monitor.OnTrialCompleted += (o, e) => + // set monitor + TrialResultMonitor monitor = null; + _experiment.SetMonitor((provider) => { - var detail = BestResultUtil.ToRunDetail(Context, e); - progressHandler?.Report(detail); - }; - - _experiment.SetMonitor(monitor); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); + _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -188,24 +200,32 @@ public override ExperimentResult Execute(IDataView public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); + _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var pipeline = this.CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); - var monitor = new TrialResultMonitor(Context); - monitor.OnTrialCompleted += (o, e) => + // set monitor + TrialResultMonitor monitor = null; + _experiment.SetMonitor((provider) => { - var detail = BestResultUtil.ToRunDetail(Context, e); - progressHandler?.Report(detail); - }; - - _experiment.SetMonitor(monitor); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); + _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -218,7 +238,7 @@ public override ExperimentResult Execute(IDataView LabelColumnName = labelColumnName, }; - return this.Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); } public override ExperimentResult Execute(IDataView trainData, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -229,32 +249,39 @@ public override ExperimentResult Execute(IDataView SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, columnInformation, preFeaturizer, progressHandler); } public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetBinaryClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); + _pipeline = CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var pipeline = this.CreateBinaryClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); - - var monitor = new TrialResultMonitor(Context); - monitor.OnTrialCompleted += (o, e) => + // set monitor + TrialResultMonitor monitor = null; + _experiment.SetMonitor((provider) => { - var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e); - - progressHandler?.Report(runDetails); - }; - - _experiment.SetMonitor(monitor); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); + + _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e)); - var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline)); + var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun, _pipeline); var result = new CrossValidationExperimentResult(runDetails, bestResult); @@ -269,7 +296,7 @@ public override CrossValidationExperimentResult Exe SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); } private protected override RunDetail GetBestRun(IEnumerable> results) @@ -282,24 +309,117 @@ private protected override CrossValidationRunDetail return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing); } - private MultiModelPipeline CreateBinaryClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) + private SweepablePipeline CreateBinaryClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { - var useSdca = this.Settings.Trainers.Contains(BinaryClassificationTrainer.SdcaLogisticRegression); - var uselbfgs = this.Settings.Trainers.Contains(BinaryClassificationTrainer.LbfgsLogisticRegression); - var useLgbm = this.Settings.Trainers.Contains(BinaryClassificationTrainer.LightGbm); - var useFastForest = this.Settings.Trainers.Contains(BinaryClassificationTrainer.FastForest); - var useFastTree = this.Settings.Trainers.Contains(BinaryClassificationTrainer.FastTree); + var useSdca = Settings.Trainers.Contains(BinaryClassificationTrainer.SdcaLogisticRegression); + var uselbfgs = Settings.Trainers.Contains(BinaryClassificationTrainer.LbfgsLogisticRegression); + var useLgbm = Settings.Trainers.Contains(BinaryClassificationTrainer.LightGbm); + var useFastForest = Settings.Trainers.Contains(BinaryClassificationTrainer.FastForest); + var useFastTree = Settings.Trainers.Contains(BinaryClassificationTrainer.FastTree); - MultiModelPipeline pipeline = new MultiModelPipeline(); if (preFeaturizer != null) { - pipeline = pipeline.Append(preFeaturizer); + return preFeaturizer.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)) + .Append(Context.Auto().BinaryClassification(labelColumnName: columnInformation.LabelColumnName, useSdca: useSdca, useFastTree: useFastTree, useLgbm: useLgbm, useLbfgs: uselbfgs, useFastForest: useFastForest, featureColumnName: Features)); } - var label = columnInformation.LabelColumnName; + else + { + return Context.Auto().Featurizer(trainData, columnInformation, Features) + .Append(Context.Auto().BinaryClassification(labelColumnName: columnInformation.LabelColumnName, useSdca: useSdca, useFastTree: useFastTree, useLgbm: useLgbm, useLbfgs: uselbfgs, useFastForest: useFastForest, featureColumnName: Features)); + } + } + } + internal class BinaryClassificationRunner : ITrialRunner + { + private readonly MLContext _context; + private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; + private readonly SweepablePipeline _pipeline; + private readonly Random _rnd; + public BinaryClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, SweepablePipeline pipeline, AutoMLExperiment.AutoMLExperimentSettings settings) + { + _context = context; + _datasetManager = datasetManager; + _metricManager = metricManager; + _pipeline = pipeline; + _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); + } + + public TrialResult Run(TrialSettings settings) + { + if (_metricManager is BinaryMetricManager metricManager) + { + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var pipeline = _pipeline.BuildFromOption(_context, parameter); + if (_datasetManager is ICrossValidateDatasetManager datasetManager) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetManager.Fold ?? 5; + var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[_rnd.Next(fold)]; + var model = res.Model; + var metric = metricManager.Metric switch + { + BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + Metrics = res.Metrics, + CrossValidationMetrics = metrics, + Pipeline = pipeline, + }; + } + + if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); + var eval = model.Transform(trainTestDatasetManager.TestDataset); + var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var metric = Enum.Parse(typeof(BinaryClassificationMetric), metricManager.MetricName) switch + { + BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + Metrics = metrics, + Pipeline = pipeline, + }; + } + } - pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, Features)); - return pipeline.Append(Context.Auto().BinaryClassification(label, useSdca: useSdca, useFastTree: useFastTree, useLgbm: useLgbm, useLbfgs: uselbfgs, useFastForest: useFastForest, featureColumnName: Features)); + throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } } } diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 7bb7c9a92e..78c503673f 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -4,8 +4,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; +using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; @@ -121,6 +124,7 @@ public sealed class MulticlassClassificationExperiment : ExperimentBase Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); // Cross val threshold for # of dataset rows -- @@ -154,24 +158,33 @@ public override ExperimentResult Execute(IDataV else { var splitData = Context.Data.TrainTestSplit(trainData); - return this.Execute(splitData.TrainSet, splitData.TestSet, columnInformation, preFeaturizer, progressHandler); + return Execute(splitData.TrainSet, splitData.TestSet, columnInformation, preFeaturizer, progressHandler); } - var pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); + _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context); - monitor.OnTrialCompleted += (o, e) => + // set monitor + TrialResultMonitor monitor = null; + _experiment.SetMonitor((provider) => { - var detail = BestResultUtil.ToRunDetail(Context, e); - progressHandler?.Report(detail); - }; - - _experiment.SetMonitor(monitor); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); + + _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -180,24 +193,34 @@ public override ExperimentResult Execute(IDataV public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, validationData); - var pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); - var monitor = new TrialResultMonitor(Context); - monitor.OnTrialCompleted += (o, e) => - { - var detail = BestResultUtil.ToRunDetail(Context, e); - progressHandler?.Report(detail); - }; + _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - _experiment.SetMonitor(monitor); + // set monitor + TrialResultMonitor monitor = null; + _experiment.SetMonitor((provider) => + { + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); + + _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e)); - var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToRunDetail(Context, e, _pipeline)); + var bestRun = BestResultUtil.ToRunDetail(Context, monitor.BestRun, _pipeline); var result = new ExperimentResult(runDetails, bestRun); return result; @@ -210,7 +233,7 @@ public override ExperimentResult Execute(IDataV LabelColumnName = labelColumnName, }; - return this.Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler); } public override ExperimentResult Execute(IDataView trainData, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -221,32 +244,40 @@ public override ExperimentResult Execute(IDataV SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, columnInformation, preFeaturizer, progressHandler); } public override CrossValidationExperimentResult Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; - _experiment.SetEvaluateMetric(Settings.OptimizingMetric, label); + _experiment.SetMulticlassClassificationMetric(Settings.OptimizingMetric, label); _experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds); _experiment.SetDataset(trainData, (int)numberOfCVFolds); - var pipeline = this.CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); - _experiment.SetPipeline(pipeline); + _pipeline = CreateMulticlassClassificationPipeline(trainData, columnInformation, preFeaturizer); + _experiment.SetPipeline(_pipeline); - var monitor = new TrialResultMonitor(Context); - monitor.OnTrialCompleted += (o, e) => + // set monitor + TrialResultMonitor monitor = null; + _experiment.SetMonitor((provider) => { - var runDetails = BestResultUtil.ToCrossValidationRunDetail(Context, e); - - progressHandler?.Report(runDetails); - }; - - _experiment.SetMonitor(monitor); + var channel = provider.GetService(); + var pipeline = provider.GetService(); + monitor = new TrialResultMonitor(channel, pipeline); + monitor.OnTrialCompleted += (o, e) => + { + var detail = BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline); + progressHandler?.Report(detail); + }; + + return monitor; + }); + + _experiment.SetTrialRunner(); _experiment.Run(); - var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e)); - var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun); + var runDetails = monitor.RunDetails.Select(e => BestResultUtil.ToCrossValidationRunDetail(Context, e, _pipeline)); + var bestResult = BestResultUtil.ToCrossValidationRunDetail(Context, monitor.BestRun, _pipeline); var result = new CrossValidationExperimentResult(runDetails, bestResult); @@ -261,7 +292,7 @@ public override CrossValidationExperimentResult SamplingKeyColumnName = samplingKeyColumn, }; - return this.Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); + return Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler); } private protected override CrossValidationRunDetail GetBestCrossValRun(IEnumerable> results) @@ -274,15 +305,15 @@ private protected override RunDetail GetBestRun return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing); } - private MultiModelPipeline CreateMulticlassClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) + private SweepablePipeline CreateMulticlassClassificationPipeline(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null) { - var useSdca = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.SdcaMaximumEntropy); - var uselbfgs = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); - var useLgbm = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.LightGbm); - var useFastForest = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.FastForestOva); - var useFastTree = this.Settings.Trainers.Contains(MulticlassClassificationTrainer.FastTreeOva); + var useSdca = Settings.Trainers.Contains(MulticlassClassificationTrainer.SdcaMaximumEntropy); + var uselbfgs = Settings.Trainers.Contains(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); + var useLgbm = Settings.Trainers.Contains(MulticlassClassificationTrainer.LightGbm); + var useFastForest = Settings.Trainers.Contains(MulticlassClassificationTrainer.FastForestOva); + var useFastTree = Settings.Trainers.Contains(MulticlassClassificationTrainer.FastTreeOva); - MultiModelPipeline pipeline = new MultiModelPipeline(); + SweepablePipeline pipeline = new SweepablePipeline(); if (preFeaturizer != null) { pipeline = pipeline.Append(preFeaturizer); @@ -298,4 +329,100 @@ private MultiModelPipeline CreateMulticlassClassificationPipeline(IDataView trai return pipeline; } } + + + internal class MulticlassClassificationRunner : ITrialRunner + { + private readonly MLContext _context; + private readonly IDatasetManager _datasetManager; + private readonly IMetricManager _metricManager; + private readonly SweepablePipeline _pipeline; + private readonly Random _rnd; + + public MulticlassClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, SweepablePipeline pipeline, AutoMLExperiment.AutoMLExperimentSettings settings) + { + _context = context; + _datasetManager = datasetManager; + _metricManager = metricManager; + _pipeline = pipeline; + _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); + } + + public TrialResult Run(TrialSettings settings) + { + if (_metricManager is MultiClassMetricManager metricManager) + { + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var pipeline = _pipeline.BuildFromOption(_context, parameter); + if (_datasetManager is ICrossValidateDatasetManager datasetManager) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetManager.Fold ?? 5; + var metrics = _context.MulticlassClassification.CrossValidate(datasetManager.Dataset, pipeline, fold, metricManager.LabelColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[_rnd.Next(fold)]; + var model = res.Model; + var metric = metricManager.Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, + _ => throw new NotImplementedException($"{metricManager.MetricName} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + Metrics = res.Metrics, + CrossValidationMetrics = metrics, + Pipeline = pipeline, + }; + } + + if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var model = pipeline.Fit(trainTestDatasetManager.TrainDataset); + var eval = model.Transform(trainTestDatasetManager.TestDataset); + var metrics = _context.MulticlassClassification.Evaluate(eval, metricManager.LabelColumn, predictedLabelColumnName: metricManager.PredictedColumn); + + var metric = metricManager.Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, + _ => throw new NotImplementedException($"{metricManager.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + Metrics = metrics, + Pipeline = pipeline, + }; + } + } + + throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); + } + } } diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index 006e35ee6e..506037d549 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -6,59 +6,48 @@ namespace Microsoft.ML.AutoML { public static class SweepableExtension { - public static SweepableEstimatorPipeline Append(this IEstimator estimator, SweepableEstimator estimator1) + public static SweepablePipeline Append(this IEstimator estimator, SweepableEstimator estimator1) { - return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); + return new SweepablePipeline().Append(estimator).Append(estimator1); } - public static SweepableEstimatorPipeline Append(this SweepableEstimatorPipeline pipeline, IEstimator estimator1) + public static SweepablePipeline Append(this SweepablePipeline pipeline, IEstimator estimator) { - return pipeline.Append(new SweepableEstimator((context, parameter) => estimator1, new SearchSpace.SearchSpace())); + return pipeline.Append(new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace())); } - public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, SweepableEstimator estimator1) + public static SweepablePipeline Append(this SweepableEstimator estimator, SweepablePipeline estimator1) { - return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); + return new SweepablePipeline().Append(estimator).Append(estimator1); } - public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, IEstimator estimator1) + public static SweepablePipeline Append(this SweepableEstimator estimator, IEstimator estimator1) { - return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); + return new SweepablePipeline().Append(estimator).Append(estimator1); } - public static MultiModelPipeline Append(this IEstimator estimator, params SweepableEstimator[] estimators) + public static SweepablePipeline Append(this IEstimator estimator, SweepablePipeline pipeline) { var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); - var multiModelPipeline = new MultiModelPipeline().Append(sweepableEstimator).Append(estimators); + var res = new SweepablePipeline().Append(sweepableEstimator).Append(pipeline); - return multiModelPipeline; + return res; } - public static MultiModelPipeline Append(this MultiModelPipeline pipeline, IEstimator estimator) + public static SweepablePipeline Append(this SweepableEstimator estimator, params SweepableEstimator[] estimators) { - var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); - var multiModelPipeline = pipeline.Append(sweepableEstimator); - - return multiModelPipeline; - } + var pipeline = new SweepablePipeline(); + pipeline = pipeline.Append(estimator); - public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) - { - var multiModelPipeline = new MultiModelPipeline(); - foreach (var estimator in pipeline.Estimators) - { - multiModelPipeline = multiModelPipeline.Append(estimator); - } - - return multiModelPipeline.Append(estimators); + return pipeline.Append(estimators); } - public static MultiModelPipeline Append(this SweepableEstimator estimator, params SweepableEstimator[] estimators) + public static SweepablePipeline Append(this IEstimator estimator, params SweepableEstimator[] estimators) { - var multiModelPipeline = new MultiModelPipeline(); - multiModelPipeline = multiModelPipeline.Append(estimator); + var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); + var pipeline = new SweepablePipeline().Append(sweepableEstimator).Append(estimators); - return multiModelPipeline.Append(estimators); + return pipeline; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index afc8dba450..e142579e14 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -7,192 +7,162 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; +using Microsoft.ML.SearchSpace; using static Microsoft.ML.DataOperationsCatalog; namespace Microsoft.ML.AutoML { public class AutoMLExperiment { + internal const string PipelineSearchspaceName = "_pipeline_"; private readonly AutoMLExperimentSettings _settings; private readonly MLContext _context; private double _bestError = double.MaxValue; private TrialResult _bestTrialResult = null; private readonly IServiceCollection _serviceCollection; + private CancellationTokenSource _cts; public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { _context = context; _settings = settings; + + if (_settings.Seed == null) + { + _settings.Seed = ((IHostEnvironmentInternal)_context.Model.GetEnvironment()).Seed; + } + + if (_settings.SearchSpace == null) + { + _settings.SearchSpace = new SearchSpace.SearchSpace(); + } + _serviceCollection = new ServiceCollection(); } private void InitializeServiceCollection() { - _serviceCollection.TryAddSingleton(_context); + _serviceCollection.TryAddTransient((provider) => + { + var context = new MLContext(_settings.Seed); + _cts.Token.Register(() => + { + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + context.CancelExecution(); + }); + + return context; + }); _serviceCollection.TryAddSingleton(_settings); - _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddSingleton(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddTransient(); - _serviceCollection.TryAddScoped(); - _serviceCollection.TryAddScoped(); + _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); + } + + private void Initialize() + { + _cts = new CancellationTokenSource(); + InitializeServiceCollection(); } + internal IServiceCollection ServiceCollection { get => _serviceCollection; } + public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) { _settings.MaxExperimentTimeInSeconds = trainingTimeInSeconds; return this; } - public AutoMLExperiment SetDataset(IDataView train, IDataView test) + public AutoMLExperiment SetIsMaximize(bool isMaximize) { - var datasetManager = new TrainTestDatasetManager() - { - TrainDataset = train, - TestDataset = test - }; - - _serviceCollection.AddSingleton(datasetManager); - + _settings.IsMaximize = isMaximize; return this; } - public AutoMLExperiment SetDataset(TrainTestData trainTestSplit) + public AutoMLExperiment AddSearchSpace(string key, SearchSpace.SearchSpace searchSpace) { - SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet); + _settings.SearchSpace[key] = searchSpace; return this; } - public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) + public AutoMLExperiment SetMonitor(TMonitor monitor) + where TMonitor : class, IMonitor { - var datasetManager = new CrossValidateDatasetManager() - { - Dataset = dataset, - Fold = fold, - }; - - _serviceCollection.AddSingleton(datasetManager); + _serviceCollection.AddSingleton(monitor); return this; } - public AutoMLExperiment SetTunerFactory() - where TTunerFactory : ITunerFactory + public AutoMLExperiment SetMonitor() + where TMonitor : class, IMonitor { - var descriptor = new ServiceDescriptor(typeof(ITunerFactory), typeof(TTunerFactory), ServiceLifetime.Singleton); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(); return this; } - public AutoMLExperiment SetMonitor(IMonitor monitor) + public AutoMLExperiment SetMonitor(Func factory) + where TMonitor : class, IMonitor { - var descriptor = new ServiceDescriptor(typeof(IMonitor), monitor); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddSingleton(factory); return this; } - public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) + public AutoMLExperiment SetTrialRunner(TTrialRunner runner) + where TTrialRunner : class, ITrialRunner { - _settings.Pipeline = pipeline; - return this; - } + _serviceCollection.AddSingleton(runner); - public AutoMLExperiment SetIsMaximizeMetric(bool isMaximize) - { - _settings.IsMaximizeMetric = isMaximize; return this; } - public AutoMLExperiment SetTrialRunner(ITrialRunner runner) + public AutoMLExperiment SetTrialRunner(Func factory) + where TTrialRunner : class, ITrialRunner { - var factory = new CustomRunnerFactory(runner); - var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), factory); - if (_serviceCollection.Contains(descriptor)) - { - _serviceCollection.Replace(descriptor); - } - else - { - _serviceCollection.Add(descriptor); - } + _serviceCollection.AddTransient(factory); return this; } - public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) + public AutoMLExperiment SetTrialRunner() + where TTrialRunner : class, ITrialRunner { - var res = new MultiModelPipeline(); - foreach (var e in pipeline.Estimators) - { - res = res.Append(e); - } - - SetPipeline(res); + _serviceCollection.AddTransient(); return this; } - public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + public AutoMLExperiment SetTuner(TTuner proposer) + where TTuner : class, ITuner { - var metricManager = new BinaryMetricManager() - { - Metric = metric, - PredictedColumn = predictedColumn, - LabelColumn = labelColumn, - }; - _serviceCollection.AddSingleton(metricManager); - SetIsMaximizeMetric(metricManager.IsMaximize); - - return this; + return this.SetTuner((service) => proposer); } - public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "PredictedLabel") + public AutoMLExperiment SetTuner(Func factory) + where TTuner : class, ITuner { - var metricManager = new MultiClassMetricManager() + var descriptor = ServiceDescriptor.Singleton(factory); + + if (_serviceCollection.Contains(descriptor)) + { + _serviceCollection.Replace(descriptor); + } + else { - Metric = metric, - PredictedColumn = predictedColumn, - LabelColumn = labelColumn, - }; - _serviceCollection.AddSingleton(metricManager); - SetIsMaximizeMetric(metricManager.IsMaximize); + _serviceCollection.Add(descriptor); + } return this; } - public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelColumn = "Label", string scoreColumn = "Score") + public AutoMLExperiment SetTuner() + where TTuner : class, ITuner { - var metricManager = new RegressionMetricManager() - { - Metric = metric, - ScoreColumn = scoreColumn, - LabelColumn = labelColumn, - }; - _serviceCollection.AddSingleton(metricManager); - SetIsMaximizeMetric(metricManager.IsMaximize); + _serviceCollection.AddSingleton(); return this; } @@ -215,71 +185,60 @@ public TrialResult Run() public async Task RunAsync(CancellationToken ct = default) { ValidateSettings(); - var cts = new CancellationTokenSource(); + Initialize(); _settings.CancellationToken = ct; - cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); - _settings.CancellationToken.Register(() => cts.Cancel()); - cts.Token.Register(() => - { - // only force-canceling running trials when there's completed trials. - // otherwise, wait for the current running trial to be completed. - if (_bestTrialResult != null) - _context.CancelExecution(); - }); - - InitializeServiceCollection(); + _cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); + _settings.CancellationToken.Register(() => _cts.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); var trialNum = 0; - var pipelineProposer = serviceProvider.GetService(); - var hyperParameterProposer = serviceProvider.GetService(); - var runnerFactory = serviceProvider.GetService(); - + var tuner = serviceProvider.GetService(); + Contracts.Assert(tuner != null, "tuner can't be null"); while (true) { - if (cts.Token.IsCancellationRequested) + if (_cts.Token.IsCancellationRequested) { break; } var setting = new TrialSettings() { - ExperimentSettings = _settings, TrialId = trialNum++, + Parameter = Parameter.CreateNestedParameter(), }; - setting = pipelineProposer.Propose(setting); - setting = hyperParameterProposer.Propose(setting); - monitor.ReportRunningTrial(setting); - var runner = runnerFactory.CreateTrialRunner(); + var parameter = tuner.Propose(setting); + setting.Parameter = parameter; + monitor?.ReportRunningTrial(setting); + var runner = serviceProvider.GetRequiredService(); try { - var trialResult = runner.Run(setting, serviceProvider); - monitor.ReportCompletedTrial(trialResult); - hyperParameterProposer.Update(setting, trialResult); - pipelineProposer.Update(setting, trialResult); + var trialResult = runner.Run(setting); + monitor?.ReportCompletedTrial(trialResult); + tuner.Update(trialResult); - var error = _settings.IsMaximizeMetric ? 1 - trialResult.Metric : trialResult.Metric; + var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; if (error < _bestError) { _bestTrialResult = trialResult; _bestError = error; - monitor.ReportBestTrial(trialResult); + monitor?.ReportBestTrial(trialResult); } } + catch (OperationCanceledException) + { + break; + } catch (Exception ex) { - if (cts.Token.IsCancellationRequested) - { - break; - } - else + monitor?.ReportFailTrial(setting, ex); + + if (!_cts.IsCancellationRequested && _bestTrialResult == null) { // TODO // it's questionable on whether to abort the entire training process // for a single fail trial. We should make it an option and only exit // when error is fatal (like schema mismatch). - monitor.ReportFailTrial(setting, ex); throw; } } @@ -303,11 +262,11 @@ private void ValidateSettings() public class AutoMLExperimentSettings : ExperimentSettings { - public MultiModelPipeline Pipeline { get; set; } - public int? Seed { get; set; } - public bool IsMaximizeMetric { get; set; } + public SearchSpace.SearchSpace SearchSpace { get; set; } + + public bool IsMaximize { get; set; } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs index ac057be17d..ee752f3e07 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetManager.cs @@ -12,14 +12,28 @@ public interface IDatasetManager { } - public class TrainTestDatasetManager : IDatasetManager + internal interface ICrossValidateDatasetManager + { + int? Fold { get; set; } + + IDataView Dataset { get; set; } + } + + internal interface ITrainTestDatasetManager + { + IDataView TrainDataset { get; set; } + + IDataView TestDataset { get; set; } + } + + internal class TrainTestDatasetManager : IDatasetManager, ITrainTestDatasetManager { public IDataView TrainDataset { get; set; } public IDataView TestDataset { get; set; } } - public class CrossValidateDatasetManager : IDatasetManager + internal class CrossValidateDatasetManager : IDatasetManager, ICrossValidateDatasetManager { public IDataView Dataset { get; set; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs index 84316b2ba5..3c25268bfb 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricManager.cs @@ -9,13 +9,27 @@ namespace Microsoft.ML.AutoML /// /// Interface for metric manager. /// - internal interface IMetricManager + public interface IMetricManager { bool IsMaximize { get; } } - internal class BinaryMetricManager : IMetricManager + public interface IEvaluateMetricManager : IMetricManager { + double Evaluate(MLContext context, IDataView eval); + + string MetricName { get; } + } + + internal class BinaryMetricManager : IEvaluateMetricManager + { + public BinaryMetricManager(BinaryClassificationMetric metric, string labelColumn, string predictedColumn) + { + Metric = metric; + PredictedColumn = predictedColumn; + LabelColumn = labelColumn; + } + public BinaryClassificationMetric Metric { get; set; } public string PredictedColumn { get; set; } @@ -34,9 +48,29 @@ internal class BinaryMetricManager : IMetricManager BinaryClassificationMetric.F1Score => true, _ => throw new NotImplementedException(), }; + + public string MetricName => Metric.ToString(); + + public double Evaluate(MLContext context, IDataView eval) + { + var metric = context.BinaryClassification.EvaluateNonCalibrated(eval, labelColumnName: LabelColumn, predictedLabelColumnName: PredictedColumn); + + return Metric switch + { + BinaryClassificationMetric.Accuracy => metric.Accuracy, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metric.AreaUnderPrecisionRecallCurve, + BinaryClassificationMetric.AreaUnderRocCurve => metric.AreaUnderRocCurve, + BinaryClassificationMetric.PositivePrecision => metric.PositivePrecision, + BinaryClassificationMetric.NegativePrecision => metric.NegativePrecision, + BinaryClassificationMetric.NegativeRecall => metric.NegativeRecall, + BinaryClassificationMetric.PositiveRecall => metric.PositivePrecision, + BinaryClassificationMetric.F1Score => metric.F1Score, + _ => throw new NotImplementedException(), + }; + } } - internal class MultiClassMetricManager : IMetricManager + internal class MultiClassMetricManager : IEvaluateMetricManager { public MulticlassClassificationMetric Metric { get; set; } @@ -53,9 +87,26 @@ internal class MultiClassMetricManager : IMetricManager MulticlassClassificationMetric.TopKAccuracy => true, _ => throw new NotImplementedException(), }; + + public string MetricName => Metric.ToString(); + + public double Evaluate(MLContext context, IDataView eval) + { + var metric = context.MulticlassClassification.Evaluate(eval, labelColumnName: LabelColumn, predictedLabelColumnName: PredictedColumn); + + return Metric switch + { + MulticlassClassificationMetric.MacroAccuracy => metric.MacroAccuracy, + MulticlassClassificationMetric.MicroAccuracy => metric.MicroAccuracy, + MulticlassClassificationMetric.LogLoss => metric.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metric.LogLossReduction, + MulticlassClassificationMetric.TopKAccuracy => metric.TopKAccuracy, + _ => throw new NotImplementedException(), + }; + } } - internal class RegressionMetricManager : IMetricManager + internal class RegressionMetricManager : IEvaluateMetricManager { public RegressionMetric Metric { get; set; } @@ -71,5 +122,21 @@ internal class RegressionMetricManager : IMetricManager RegressionMetric.MeanAbsoluteError => false, _ => throw new NotImplementedException(), }; + + public string MetricName => Metric.ToString(); + + public double Evaluate(MLContext context, IDataView eval) + { + var metric = context.Regression.Evaluate(eval, LabelColumn, ScoreColumn); + + return Metric switch + { + RegressionMetric.RSquared => metric.RSquared, + RegressionMetric.RootMeanSquaredError => metric.RootMeanSquaredError, + RegressionMetric.MeanSquaredError => metric.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => metric.MeanAbsoluteError, + _ => throw new NotImplementedException(), + }; + } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 122ea4f31c..fb52ebe686 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -22,49 +22,48 @@ public interface IMonitor void ReportRunningTrial(TrialSettings setting); } - // this monitor redirects output result to context.log + /// + /// monitor that redirect output results to context.Log channel. + /// internal class MLContextMonitor : IMonitor { - private readonly MLContext _context; - private readonly IServiceProvider _serviceProvider; private readonly IChannel _logger; private readonly List _completedTrials; - - public MLContextMonitor(MLContext context, IServiceProvider provider = null) + private readonly SweepablePipeline _pipeline; + public MLContextMonitor(IChannel logger, SweepablePipeline pipeline) { - _context = context; - _serviceProvider = provider; - _logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); + _logger = logger; _completedTrials = new List(); + _pipeline = pipeline; } public virtual void ReportBestTrial(TrialResult result) { - _logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + _logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)}"); } public virtual void ReportCompletedTrial(TrialResult result) { - _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline} - Duration: {result.DurationInMilliseconds}"); + _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds}"); _completedTrials.Add(result); } public virtual void ReportFailTrial(TrialSettings settings, Exception exception = null) { - _logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {settings.Pipeline}"); + _logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {_pipeline.ToString(settings.Parameter)}"); } public virtual void ReportRunningTrial(TrialSettings setting) { - _logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {setting.Pipeline}"); + _logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {_pipeline.ToString(setting.Parameter)}"); } } internal class TrialResultMonitor : MLContextMonitor where TMetrics : class { - public TrialResultMonitor(MLContext context) - : base(context, null) + public TrialResultMonitor(IChannel channel, SweepablePipeline pipeline) + : base(channel, pipeline) { this.RunDetails = new List>(); } @@ -91,10 +90,10 @@ public override void ReportBestTrial(TrialResult result) public override void ReportCompletedTrial(TrialResult result) { base.ReportCompletedTrial(result); - if (result is TrialResult binaryClassificationResult) + if (result is TrialResult metricResult) { - RunDetails.Add(binaryClassificationResult); - OnTrialCompleted?.Invoke(this, binaryClassificationResult); + RunDetails.Add(metricResult); + OnTrialCompleted?.Invoke(this, metricResult); } else { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs new file mode 100644 index 0000000000..c19ae220b6 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + /// + /// interface for all trial runners. + /// + public interface ITrialRunner + { + TrialResult Run(TrialSettings settings); + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs new file mode 100644 index 0000000000..f869df6ba4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + internal class SweepablePipelineRunner : ITrialRunner + { + private readonly MLContext _mLContext; + private readonly IEvaluateMetricManager _metricManager; + private readonly IDatasetManager _datasetManager; + private readonly SweepablePipeline _pipeline; + + public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager) + { + _mLContext = context; + _metricManager = metricManager; + _pipeline = pipeline; + _datasetManager = datasetManager; + } + + public TrialResult Run(TrialSettings settings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; + var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); + + if (_datasetManager is ICrossValidateDatasetManager crossValidateDatasetManager) + { + var datasetSplit = _mLContext.Data.CrossValidationSplit(crossValidateDatasetManager.Dataset, crossValidateDatasetManager.Fold ?? 5); + var metrics = new List(); + var models = new List(); + foreach (var split in datasetSplit) + { + var model = mlnetPipeline.Fit(split.TrainSet); + var eval = model.Transform(split.TestSet); + metrics.Add(_metricManager.Evaluate(_mLContext, eval)); + models.Add(model); + } + + stopWatch.Stop(); + + return new TrialResult + { + Metric = metrics.Average(), + Model = models.First(), + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + TrialSettings = settings, + }; + } + + if (_datasetManager is ITrainTestDatasetManager trainTestDatasetManager) + { + var model = mlnetPipeline.Fit(trainTestDatasetManager.TrainDataset); + var eval = model.Transform(trainTestDatasetManager.TestDataset); + var metric = _metricManager.Evaluate(_mLContext, eval); + stopWatch.Stop(); + + return new TrialResult + { + Metric = metric, + Model = model, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + TrialSettings = settings, + }; + } + + throw new ArgumentException("IDatasetManager must be either ITrainTestDatasetManager or ICrossValidationDatasetManager"); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index b3803215d3..f08e499d2f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -36,5 +36,7 @@ internal class TrialResult : TrialResult public bool IsSucceed { get => Exception == null; } public bool IsCrossValidation { get => CrossValidationMetrics == null; } + + public EstimatorChain Pipeline { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs deleted file mode 100644 index 596729bb61..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ /dev/null @@ -1,339 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Diagnostics; -using System.Linq; -using Microsoft.ML.Data; - -namespace Microsoft.ML.AutoML -{ - /// - /// interface for all trial runners. - /// - public interface ITrialRunner - { - TrialResult Run(TrialSettings settings, IServiceProvider provider = null); - } - - internal class BinaryClassificationCVRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public BinaryClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _datasetManager = datasetManager; - _metricManager = metricManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is BinaryMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var metrics = _context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch - { - BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - Metrics = res.Metrics, - CrossValidationMetrics = metrics, - }; - } - - throw new ArgumentException(); - } - } - - internal class BinaryClassificationTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public BinaryClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is BinaryMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); - - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var metric = metricSettings.Metric switch - { - BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, - BinaryClassificationMetric.Accuracy => metrics.Accuracy, - BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, - BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - Metrics = metrics, - }; - } - - throw new ArgumentException(); - } - } - - internal class MultiClassificationTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public MultiClassificationTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is MultiClassMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.MulticlassClassification.Evaluate(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); - - var metric = metricSettings.Metric switch - { - MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, - MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, - MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, - MulticlassClassificationMetric.LogLoss => metrics.LogLoss, - MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new TrialResult() - { - Metrics = metrics, - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - } - - throw new ArgumentException(); - } - } - - internal class MultiClassificationCVRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public MultiClassificationCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is MultiClassMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var metrics = _context.MulticlassClassification.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch - { - MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, - MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, - MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, - MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, - MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - CrossValidationMetrics = metrics, - Metrics = res.Metrics, - }; - } - - throw new ArgumentException(); - } - } - - internal class RegressionTrainTestRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public RegressionTrainTestRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - if (_datasetManager is TrainTestDatasetManager datasetSettings - && _metricManager is RegressionMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var model = pipeline.Fit(datasetSettings.TrainDataset); - var eval = model.Transform(datasetSettings.TestDataset); - var metrics = _context.Regression.Evaluate(eval, metricSettings.LabelColumn, scoreColumnName: metricSettings.ScoreColumn); - - var metric = metricSettings.Metric switch - { - RegressionMetric.RootMeanSquaredError => metrics.RootMeanSquaredError, - RegressionMetric.RSquared => metrics.RSquared, - RegressionMetric.MeanSquaredError => metrics.MeanSquaredError, - RegressionMetric.MeanAbsoluteError => metrics.MeanAbsoluteError, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - } - - throw new ArgumentException(); - } - } - - internal class RegressionCVRunner : ITrialRunner - { - private readonly MLContext _context; - private readonly IDatasetManager _datasetManager; - private readonly IMetricManager _metricManager; - - public RegressionCVRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager) - { - _context = context; - _metricManager = metricManager; - _datasetManager = datasetManager; - } - - public TrialResult Run(TrialSettings settings, IServiceProvider provider) - { - var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); - if (_datasetManager is CrossValidateDatasetManager datasetSettings - && _metricManager is RegressionMetricManager metricSettings) - { - var stopWatch = new Stopwatch(); - stopWatch.Start(); - var fold = datasetSettings.Fold ?? 5; - - var pipeline = settings.Pipeline.BuildTrainingPipeline(_context, settings.Parameter); - var metrics = _context.Regression.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. - var res = metrics[rnd.Next(fold)]; - var model = res.Model; - var metric = metricSettings.Metric switch - { - RegressionMetric.RootMeanSquaredError => res.Metrics.RootMeanSquaredError, - RegressionMetric.RSquared => res.Metrics.RSquared, - RegressionMetric.MeanSquaredError => res.Metrics.MeanSquaredError, - RegressionMetric.MeanAbsoluteError => res.Metrics.MeanAbsoluteError, - _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), - }; - - stopWatch.Stop(); - - return new TrialResult() - { - Metric = metric, - Model = model, - TrialSettings = settings, - DurationInMilliseconds = stopWatch.ElapsedMilliseconds, - }; - } - - throw new ArgumentException(); - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs deleted file mode 100644 index 8bba47d321..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using Microsoft.Extensions.DependencyInjection; - -#nullable enable -namespace Microsoft.ML.AutoML -{ - /// - /// interface for trial runner factory. - /// - public interface ITrialRunnerFactory - { - ITrialRunner? CreateTrialRunner(); - } - - internal class CustomRunnerFactory : ITrialRunnerFactory - { - private readonly ITrialRunner _instance; - - public CustomRunnerFactory(ITrialRunner runner) - { - _instance = runner; - } - - public ITrialRunner? CreateTrialRunner() - { - return _instance; - } - } - - internal class TrialRunnerFactory : ITrialRunnerFactory - { - private readonly IServiceProvider _provider; - - public TrialRunnerFactory(IServiceProvider provider) - { - _provider = provider; - } - - public ITrialRunner? CreateTrialRunner() - { - var datasetManager = _provider.GetService(); - var metricManager = _provider.GetService(); - - ITrialRunner? runner = (datasetManager, metricManager) switch - { - (CrossValidateDatasetManager, BinaryMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, BinaryMetricManager) => _provider.GetService(), - (CrossValidateDatasetManager, MultiClassMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, MultiClassMetricManager) => _provider.GetService(), - (CrossValidateDatasetManager, RegressionMetricManager) => _provider.GetService(), - (TrainTestDatasetManager, RegressionMetricManager) => _provider.GetService(), - _ => throw new NotImplementedException(), - }; - - return runner; - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs index 19294ffde9..38d14c2a6d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs @@ -10,12 +10,6 @@ public class TrialSettings { public int TrialId { get; set; } - public SweepableEstimatorPipeline Pipeline { get; set; } - - public string Schema { get; set; } - public Parameter Parameter { get; set; } - - public AutoMLExperiment.AutoMLExperimentSettings ExperimentSettings { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs deleted file mode 100644 index ee091027b2..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.DependencyInjection; - -namespace Microsoft.ML.AutoML -{ - internal class HyperParameterProposer : ITrialSettingsProposer - { - private readonly Dictionary _tuners; - private readonly IServiceProvider _provider; - - public HyperParameterProposer(IServiceProvider provider) - { - _tuners = new Dictionary(); - _provider = provider; - } - - public TrialSettings Propose(TrialSettings settings) - { - var tunerFactory = _provider.GetService(); - if (!_tuners.ContainsKey(settings.Schema)) - { - var t = tunerFactory.CreateTuner(settings); - _tuners.Add(settings.Schema, t); - } - - var tuner = _tuners[settings.Schema]; - var parameter = tuner.Propose(settings); - settings.Parameter = parameter; - - return settings; - } - - public void Update(TrialSettings settings, TrialResult result) - { - var schema = settings.Schema; - if (_tuners.TryGetValue(schema, out var tuner)) - { - tuner.Update(result); - } - } - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs deleted file mode 100644 index ad73331fab..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.ML.AutoML -{ - internal interface ITrialSettingsProposer - { - TrialSettings Propose(TrialSettings settings); - - void Update(TrialSettings parameter, TrialResult result); - } - - internal interface ISavableProposer : ITrialSettingsProposer - { - void SaveStatusToFile(string fileName); - - void LoadStatusFromFile(string fileName); - } -} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs deleted file mode 100644 index 6acc5f37e6..0000000000 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using Microsoft.Extensions.DependencyInjection; - -namespace Microsoft.ML.AutoML -{ - /// - /// interface for all tuner factories. - /// - public interface ITunerFactory - { - ITuner CreateTuner(TrialSettings settings); - } - - internal class CostFrugalTunerFactory : ITunerFactory - { - private readonly IServiceProvider _provider; - - public CostFrugalTunerFactory(IServiceProvider provider) - { - _provider = provider; - } - - public ITuner CreateTuner(TrialSettings settings) - { - var experimentSetting = _provider.GetService(); - var searchSpace = settings.Pipeline.SearchSpace; - var initParameter = settings.Pipeline.Parameter; - var isMaximize = experimentSetting.IsMaximizeMetric; - - return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); - } - } - - internal class RandomTunerFactory : ITunerFactory - { - private readonly IServiceProvider _provider; - - public RandomTunerFactory(IServiceProvider provider) - { - _provider = provider; - } - - public ITuner CreateTuner(TrialSettings settings) - { - var searchSpace = settings.Pipeline.SearchSpace; - - return new RandomSearchTuner(searchSpace); - } - } - - internal class GridSearchTunerFactory : ITunerFactory - { - private readonly IServiceProvider _provider; - - public GridSearchTunerFactory(IServiceProvider provider) - { - _provider = provider; - } - - public ITuner CreateTuner(TrialSettings settings) - { - var searchSpace = settings.Pipeline.SearchSpace; - - return new GridSearchTuner(searchSpace); - } - } -} diff --git a/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json b/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json index 2794e3c961..a0f6e1fd2f 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json +++ b/src/Microsoft.ML.AutoML/CodeGen/lgbm_search_space.json @@ -7,7 +7,7 @@ "type": "integer", "default": 4, "search_space": { - "log_base": true, + "log_base": false, "max": 32768, "min": 4 } @@ -39,7 +39,7 @@ "search_space": { "min": 4, "max": 32768, - "log_base": true + "log_base": false } }, { diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs index 5e7a6f681a..01a0a2935c 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/ISweepable.cs @@ -9,12 +9,12 @@ namespace Microsoft.ML.AutoML { - internal interface ISweepable + public interface ISweepable { public SearchSpace.SearchSpace SearchSpace { get; } } - internal interface ISweepable : ISweepable + public interface ISweepable : ISweepable where T : IEstimator { public T BuildFromOption(MLContext context, Parameter parameter); diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs index eb1b58d1a1..885a726b79 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs @@ -10,14 +10,14 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(MultiModelPipelineConverter))] - public class MultiModelPipeline + internal class MultiModelPipeline { private static readonly StringEntity _nilStringEntity = new StringEntity("Nil"); private static readonly EstimatorEntity _nilSweepableEntity = new EstimatorEntity(null); private readonly Dictionary _estimators; private readonly Entity _schema; - public MultiModelPipeline() + internal MultiModelPipeline() { _estimators = new Dictionary(); _schema = null; diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs index f04b7ae63b..76e5682ba5 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(SweepableEstimatorPipelineConverter))] - public class SweepableEstimatorPipeline + internal class SweepableEstimatorPipeline { private readonly List _estimators; diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs index 37c4ac385e..6f754d12b5 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepablePipeline.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.AutoML { [JsonConverter(typeof(SweepablePipelineConverter))] - internal class SweepablePipeline : ISweepable> + public class SweepablePipeline : ISweepable> { private readonly Entity _schema; private const string SchemaOption = "_SCHEMA_"; @@ -28,12 +28,11 @@ public SearchSpace.SearchSpace SearchSpace get { var searchSpace = new SearchSpace.SearchSpace(); - var kvPairs = _estimators.Select((e, i) => new KeyValuePair(i.ToString(), e.Value.SearchSpace)); - foreach (var kv in kvPairs) + foreach (var kv in _estimators) { if (kv.Value != null) { - searchSpace.Add(kv.Key, kv.Value); + searchSpace.Add(kv.Key, kv.Value.SearchSpace); } } @@ -85,13 +84,34 @@ internal SweepablePipeline(Dictionary estimators, En public EstimatorChain BuildFromOption(MLContext context, Parameter parameter) { _currentSchema = parameter[SchemaOption].AsType(); - var estimators = Entity.FromExpression(_currentSchema) + var pipeline = new EstimatorChain(); + var estimatorParameterPair = Entity.FromExpression(_currentSchema) .ValueEntities() .Where(e => e is StringEntity se && se.Value != "Nil") - .Select((se) => _estimators[((StringEntity)se).Value]); + .Select((se) => + { + var key = ((StringEntity)se).Value; + var estimator = _estimators[key]; + var param = parameter[key]; + return (estimator, param); + }); - var pipeline = new SweepableEstimatorPipeline(estimators); - return pipeline.BuildTrainingPipeline(context, parameter); + foreach (var kv in estimatorParameterPair) + { + pipeline = pipeline.Append(kv.estimator.BuildFromOption(context, kv.param)); + } + + return pipeline; + } + + public SweepablePipeline BuildSweepableEstimatorPipeline(string schema) + { + var entity = Entity.FromExpression(schema); + var pipelineNodes = entity.ValueEntities() + .Where(e => e is StringEntity se && se.Value != "Nil") + .ToDictionary((se) => se.ToString(), (se) => _estimators[((StringEntity)se).Value]); + + return new SweepablePipeline(pipelineNodes, entity, schema); } public SweepablePipeline Append(params ISweepable>[] sweepables) @@ -128,6 +148,27 @@ public SweepablePipeline Append(params ISweepable>[] sw return AppendEntity(false, entity); } + public string ToString(Parameter parameter) + { + if (parameter.TryGetValue(AutoMLExperiment.PipelineSearchspaceName, out var pipelineParameter)) + { + var schema = pipelineParameter["_SCHEMA_"].AsType(); + var estimatorStrings = Entity.FromExpression(_currentSchema) + .ValueEntities() + .Where(e => e is StringEntity se && se.Value != "Nil") + .Select((se) => + { + var key = ((StringEntity)se).Value; + var estimator = _estimators[key]; + return estimator.EstimatorType.ToString(); + }); + + return string.Join("=>", estimatorStrings); + } + + return string.Empty; + } + private SweepablePipeline AppendEntity(bool allowSkip, Entity entity) { var estimators = _estimators.ToDictionary(x => x.Key, x => x.Value); diff --git a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs index 949f851dff..5622c0c1ed 100644 --- a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs @@ -25,6 +25,11 @@ internal class CostFrugalTuner : ITuner private bool _initUsed = false; private double _bestMetric; + public CostFrugalTuner(AutoMLExperiment.AutoMLExperimentSettings settings, IMetricManager manager) + : this(settings.SearchSpace, settings.SearchSpace.SampleFromFeatureSpace(settings.SearchSpace.Default), !manager.IsMaximize) + { + } + public CostFrugalTuner(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true) { _searchSpace = searchSpace; diff --git a/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs new file mode 100644 index 0000000000..641087b9b7 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/EciCfoTuner.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + /// + /// propose hyper parameter using ECI index and . + /// ECI index is a way to measure the importance of a trainer. A higher ECI means a trainer + /// is more likely to be picked. + /// + public class EciCostFrugalTuner : ITuner + { + private readonly Dictionary _tuners; + private readonly PipelineProposer _pipelineProposer; + // this dictionary records the schema for each trial. + // the key is trial id, and value is the schema for that trial. + private readonly IMetricManager _metricManager; + + public EciCostFrugalTuner(SweepablePipeline sweepablePipeline, IMetricManager metricManager, AutoMLExperiment.AutoMLExperimentSettings settings) + { + _tuners = new Dictionary(); + _pipelineProposer = new PipelineProposer(sweepablePipeline, settings, metricManager); + _metricManager = metricManager; + } + + public Parameter Propose(TrialSettings settings) + { + (var searchSpace, var schema) = _pipelineProposer.ProposeSearchSpace(); + if (!_tuners.ContainsKey(schema)) + { + var t = new CostFrugalTuner(searchSpace, searchSpace.SampleFromFeatureSpace(searchSpace.Default), !_metricManager.IsMaximize); + _tuners.Add(schema, t); + } + + var tuner = _tuners[schema]; + settings.Parameter[AutoMLExperiment.PipelineSearchspaceName] = tuner.Propose(settings); + + return settings.Parameter; + } + + public void Update(TrialResult result) + { + var schema = result.TrialSettings.Parameter[AutoMLExperiment.PipelineSearchspaceName]["_SCHEMA_"].AsType(); + if (_tuners.TryGetValue(schema, out var tuner)) + { + tuner.Update(result); + } + + _pipelineProposer.Update(result, schema); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs index d060ac9ed7..e68461899b 100644 --- a/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs @@ -12,6 +12,11 @@ internal class GridSearchTuner : ITuner private readonly SearchSpace.Tuner.GridSearchTuner _tuner; private IEnumerator _enumerator; + public GridSearchTuner(AutoMLExperiment.AutoMLExperimentSettings settings) + : this(settings.SearchSpace, 10) + { + } + public GridSearchTuner(SearchSpace.SearchSpace searchSpace, int stepSize = 10) { _tuner = new SearchSpace.Tuner.GridSearchTuner(searchSpace, stepSize); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/Tuner/PipelineProposer.cs similarity index 70% rename from src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs rename to src/Microsoft.ML.AutoML/Tuner/PipelineProposer.cs index 75b5ffabd3..21f66905d2 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/Tuner/PipelineProposer.cs @@ -7,7 +7,9 @@ using System.IO; using System.Linq; using Microsoft.ML.AutoML.CodeGen; +using Microsoft.ML.SearchSpace.Option; using Newtonsoft.Json; +using Tensorflow; using static Microsoft.ML.AutoML.AutoMLExperiment; namespace Microsoft.ML.AutoML @@ -15,7 +17,7 @@ namespace Microsoft.ML.AutoML /// /// propose sweepable estimator pipeline from a group of candidates using eci in flaml (https://arxiv.org/abs/1911.04706) /// - internal class PipelineProposer : ISavableProposer + internal class PipelineProposer { private readonly Dictionary _estimatorCost; private Dictionary _learnerInitialCost; @@ -37,9 +39,11 @@ internal class PipelineProposer : ISavableProposer private double _globalBestError; private readonly Random _rand; - private MultiModelPipeline _multiModelPipeline; + private readonly SweepablePipeline _sweepablePipeline; + private readonly IMetricManager _metricManager; + private readonly string[] _pipelineSchemas; - public PipelineProposer(AutoMLExperimentSettings settings) + public PipelineProposer(SweepablePipeline sweepablePipeline, AutoMLExperimentSettings settings, IMetricManager metricManager) { // this cost is used to initialize eci when started, the smaller the number, the less cost this trainer will use at start, and more likely it will be // picked. @@ -67,26 +71,35 @@ public PipelineProposer(AutoMLExperimentSettings settings) { EstimatorType.ImageClassificationMulti, 1 }, { EstimatorType.MatrixFactorization, 1 }, }; - _rand = new Random(settings.Seed ?? 0); - _multiModelPipeline = null; + _sweepablePipeline = sweepablePipeline; + + if (settings.Seed.HasValue) + { + _rand = new Random(settings.Seed.Value); + } + else + { + _rand = new Random(); + } + + _pipelineSchemas = _sweepablePipeline.Schema.ToTerms().Select(t => t.ToString()).ToArray(); + _metricManager = metricManager; } - public TrialSettings Propose(TrialSettings settings) + public (SearchSpace.SearchSpace, string) ProposeSearchSpace() { - _multiModelPipeline = settings.ExperimentSettings.Pipeline; - _learnerInitialCost = _multiModelPipeline.PipelineIds.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _multiModelPipeline)); - var pipelineIds = _multiModelPipeline.PipelineIds; - + _learnerInitialCost = _pipelineSchemas.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _sweepablePipeline)); + string schema; if (_eci == null) { // initialize eci with the estimated cost and always start from pipeline which has lowest cost. - _eci = pipelineIds.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _multiModelPipeline)); - settings.Schema = _eci.OrderBy(kv => kv.Value).First().Key; + _eci = _pipelineSchemas.ToDictionary(kv => kv, kv => GetEstimatedCostForPipeline(kv, _sweepablePipeline)); + schema = _eci.OrderBy(kv => kv.Value).First().Key; } else { - var probabilities = pipelineIds.Select(id => _eci[id]).ToArray(); + var probabilities = _pipelineSchemas.Select(id => _eci[id]).ToArray(); probabilities = ArrayMath.Inverse(probabilities); probabilities = ArrayMath.Normalize(probabilities); @@ -96,7 +109,7 @@ public TrialSettings Propose(TrialSettings settings) // selected pipeline id index int i; - for (i = 0; i != pipelineIds.Length; ++i) + for (i = 0; i != _pipelineSchemas.Length; ++i) { sum += probabilities[i]; if (sum > randdouble) @@ -105,61 +118,16 @@ public TrialSettings Propose(TrialSettings settings) } } - settings.Schema = pipelineIds[i]; - } - - settings.Pipeline = _multiModelPipeline.BuildSweepableEstimatorPipeline(settings.Schema); - return settings; - } - - public void SaveStatusToFile(string fileName) - { - using (var writer = new FileStream(fileName, FileMode.Create)) - { - SaveStatusToStream(writer); - } - } - - public void SaveStatusToStream(Stream stream) - { - var status = new Status() - { - K1 = _k1, - K2 = _k2, - E1 = _e1, - E2 = _e2, - Eci = _eci, - GlobalBestError = _globalBestError, - }; - - using (var fileWriter = new StreamWriter(stream)) - { - var json = JsonConvert.SerializeObject(status); - fileWriter.Write(json); + schema = _pipelineSchemas[i]; } - } - public void LoadStatusFromFile(string fileName) - { - if (File.Exists(fileName)) - { - var json = File.ReadAllText(fileName); - var status = JsonConvert.DeserializeObject(json); - _k1 = status.K1; - _k2 = status.K2; - _e1 = status.E1; - _e2 = status.E2; - _eci = status.Eci; - _globalBestError = status.GlobalBestError; - } + return (_sweepablePipeline.BuildSweepableEstimatorPipeline(schema).SearchSpace, schema); } - public void Update(TrialSettings parameter, TrialResult result) + public void Update(TrialResult result, string schema) { - var schema = parameter.Schema; - var error = CaculateError(result.Metric, parameter.ExperimentSettings.IsMaximizeMetric); + var error = CaculateError(result.Metric, _metricManager.IsMaximize); var duration = result.DurationInMilliseconds / 1000; - var pipelineIds = _multiModelPipeline.PipelineIds; var isSuccess = duration != 0; // if k1 is null, it means this is the first completed trial. @@ -173,10 +141,10 @@ public void Update(TrialSettings parameter, TrialResult result) { if (_k1 == null) { - _k1 = pipelineIds.ToDictionary(id => id, id => duration * _learnerInitialCost[id] / _learnerInitialCost[schema]); + _k1 = _pipelineSchemas.ToDictionary(id => id, id => duration * _learnerInitialCost[id] / _learnerInitialCost[schema]); _k2 = _k1.ToDictionary(kv => kv.Key, kv => kv.Value); - _e1 = pipelineIds.ToDictionary(id => id, id => error); - _e2 = pipelineIds.ToDictionary(id => id, id => 1.05 * error); + _e1 = _pipelineSchemas.ToDictionary(id => id, id => error); + _e2 = _pipelineSchemas.ToDictionary(id => id, id => 1.05 * error); _globalBestError = error; } else if (error >= _e1[schema]) @@ -227,15 +195,15 @@ private double CaculateError(double loss, bool isMaximize) return isMaximize ? 1 - loss : loss; } - private double GetEstimatedCostForPipeline(string kv, MultiModelPipeline multiModelPipeline) + private double GetEstimatedCostForPipeline(string schema, SweepablePipeline pipeline) { - var entity = Entity.FromExpression(kv); + var entity = Entity.FromExpression(schema); var estimatorTypes = entity.ValueEntities().Where(v => v is StringEntity s && s.Value != "Nil") .Select(v => { var s = v as StringEntity; - var estimator = multiModelPipeline.Estimators[s.Value]; + var estimator = pipeline.Estimators[s.Value]; return estimator.EstimatorType; }); diff --git a/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs index 0a860a4730..fd63a51dfa 100644 --- a/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs @@ -12,6 +12,11 @@ internal class RandomSearchTuner : ITuner private readonly RandomTuner _tuner; private readonly SearchSpace.SearchSpace _searchSpace; + public RandomSearchTuner(AutoMLExperiment.AutoMLExperimentSettings settings) + : this(settings.SearchSpace) + { + } + public RandomSearchTuner(SearchSpace.SearchSpace searchSpace) { _tuner = new RandomTuner(); diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index 0654fc1f95..cdcf850d8f 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -96,29 +96,25 @@ public static int GetIndexOfBestScore(IEnumerable scores, bool isMetricM return isMetricMaximizing ? GetIndexOfMaxScore(scores) : GetIndexOfMinScore(scores); } - public static RunDetail ToRunDetail(MLContext context, TrialResult result) + public static RunDetail ToRunDetail(MLContext context, TrialResult result, SweepablePipeline pipeline) where TMetrics : class { - var pipeline = result.TrialSettings.Pipeline; - var trainerName = pipeline.ToString(); var parameter = result.TrialSettings.Parameter; - var estimator = pipeline.BuildTrainingPipeline(context, parameter); + var trainerName = pipeline.ToString(parameter); var modelContainer = new ModelContainer(context, result.Model); - var detail = new RunDetail(trainerName, estimator, null, modelContainer, result.Metrics, result.Exception); + var detail = new RunDetail(trainerName, result.Pipeline, null, modelContainer, result.Metrics, result.Exception); detail.RuntimeInSeconds = result.DurationInMilliseconds / 1000; return detail; } - public static CrossValidationRunDetail ToCrossValidationRunDetail(MLContext context, TrialResult result) + public static CrossValidationRunDetail ToCrossValidationRunDetail(MLContext context, TrialResult result, SweepablePipeline pipeline) where TMetrics : class { - var pipeline = result.TrialSettings.Pipeline; - var trainerName = pipeline.ToString(); var parameter = result.TrialSettings.Parameter; - var estimator = pipeline.BuildTrainingPipeline(context, parameter); + var trainerName = pipeline.ToString(parameter); var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult(new ModelContainer(context, m.Model), m.Metrics, result.Exception)); - var detail = new CrossValidationRunDetail(trainerName, estimator, null, crossValidationResult); + var detail = new CrossValidationRunDetail(trainerName, result.Pipeline, null, crossValidationResult); detail.RuntimeInSeconds = result.DurationInMilliseconds / 1000; return detail; diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_creditapproval_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_creditapproval_test.approved.txt new file mode 100644 index 0000000000..c2bea3ba98 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_creditapproval_test.approved.txt @@ -0,0 +1,86 @@ +{ + "schema": "e0 * e1 * (e2 \u002B e3) * e4", + "currentSchema": "e0 * e1 * e2 * e4", + "estimators": { + "e0": { + "estimatorType": "ReplaceMissingValues", + "parameter": { + "OutputColumnNames": [ + "Features" + ], + "InputColumnNames": [ + "Features" + ] + } + }, + "e1": { + "estimatorType": "ConvertType", + "parameter": { + "OutputColumnNames": [ + "BooleanFeatures" + ], + "InputColumnNames": [ + "BooleanFeatures" + ] + } + }, + "e2": { + "estimatorType": "OneHotEncoding", + "parameter": { + "OutputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ], + "InputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ] + } + }, + "e3": { + "estimatorType": "OneHotHashEncoding", + "parameter": { + "OutputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ], + "InputColumnNames": [ + "A1", + "A4", + "A5", + "A6", + "A7", + "A13" + ] + } + }, + "e4": { + "estimatorType": "Concatenate", + "parameter": { + "InputColumnNames": [ + "Features", + "A1", + "A4", + "A5", + "A6", + "A7", + "A13", + "BooleanFeatures" + ], + "OutputColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt index 4f137b8a8a..d2678afcc9 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * e2 * e3 * e4", + "currentSchema": "e0 * e1 * e2 * e3 * e4", "estimators": { "e0": { "estimatorType": "LoadImages", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt index e6ec8ed089..d82286552b 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "ReplaceMissingValues", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt index 3a57617a98..fe234e918c 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_newspaperchurn_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * (e2 \u002B e3) * e4 * e5", + "currentSchema": "e0 * e1 * e2 * e4 * e5", "estimators": { "e0": { "estimatorType": "ReplaceMissingValues", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt index 63a64233a4..a265687a48 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 \u002B e2) * e3", + "currentSchema": "e0 * e1 * e3", "estimators": { "e0": { "estimatorType": "ReplaceMissingValues", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt index 0b930b01aa..0a39f1f6fc 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.ImagePathFeaturizerTest.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * e2 * e3", + "currentSchema": "e0 * e1 * e2 * e3", "estimators": { "e0": { "estimatorType": "LoadImages", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt new file mode 100644 index 0000000000..4a334ab2f5 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.AppendIEstimatorToSweepabePipelineTest.approved.txt @@ -0,0 +1,34 @@ +{ + "schema": "e0 * (e1 + e2) * e3", + "currentSchema": "e0 * e1 * e3", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e2": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e3": { + "estimatorType": "Unknown", + "parameter": {} + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt index d3be549eed..58efe42258 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt index 0e14cdb320..b02699087d 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt index 0e14cdb320..b02699087d 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..b02699087d --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,85 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt index 9661d5897c..258dc76181 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", "estimators": { "e0": { "estimatorType": "FastForestBinary", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..3f6be2bbb3 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,85 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt similarity index 98% rename from test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt rename to test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt index 05949bc69e..5ea3e5977e 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt @@ -1,5 +1,6 @@ { "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", "estimators": { "e0": { "estimatorType": "Unknown", diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..5ea3e5977e --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepableEstimatorPipelineAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,89 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..5ea3e5977e --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt @@ -0,0 +1,89 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt new file mode 100644 index 0000000000..5ea3e5977e --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt @@ -0,0 +1,89 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "currentSchema": "e0 * e1 * e2", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..b02699087d --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateSweepablePipelinePipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,85 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "currentSchema": "e0 * e1", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 0.0000000002, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs index 839cf0ffe0..84365449fe 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs @@ -74,6 +74,18 @@ public void AutoFeaturizer_newspaperchurn_test() Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions)); } + [Fact] + [UseReporter(typeof(DiffReporter))] + [UseApprovalSubdirectory("ApprovalTests")] + public void AutoFeaturizer_creditapproval_test() + { + // this test verify if auto featurizer can convert vector column to vector. + var context = new MLContext(1); + var dataset = DatasetUtil.GetCreditApprovalDataView(); + var pipeline = context.Auto().Featurizer(dataset, excludeColumns: new[] { "A16" }); + Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions)); + } + [Fact] [UseReporter(typeof(DiffReporter))] [UseApprovalSubdirectory("ApprovalTests")] diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 97911f7bb1..ff2ba2474d 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -14,6 +14,7 @@ using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.TestFrameworkCommon.Attributes; +using Microsoft.VisualBasic; using Xunit; using Xunit.Abstractions; using static Microsoft.ML.DataOperationsCatalog; @@ -37,7 +38,7 @@ private void MlContextLog(object sender, LoggingEventArgs e) _markerAutoFitContextLogTest = true; } - [LightGBMFact] + [Fact] public void AutoFit_UCI_Adult_Test() { var context = new MLContext(1); @@ -45,8 +46,17 @@ public void AutoFit_UCI_Adult_Test() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + settings.Trainers.Remove(BinaryClassificationTrainer.SdcaLogisticRegression); + settings.Trainers.Remove(BinaryClassificationTrainer.LbfgsLogisticRegression); + var result = context.Auto() - .CreateBinaryClassificationExperiment(1) + .CreateBinaryClassificationExperiment(settings) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -54,7 +64,7 @@ public void AutoFit_UCI_Adult_Test() Assert.NotNull(result.BestRun.TrainerName); } - [LightGBMFact] + [Fact] public void AutoFit_UCI_Adult_Train_Test_Split_Test() { var context = new MLContext(1); @@ -63,8 +73,17 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); var dataTrainTest = context.Data.TrainTestSplit(trainData); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + settings.Trainers.Remove(BinaryClassificationTrainer.SdcaLogisticRegression); + settings.Trainers.Remove(BinaryClassificationTrainer.LbfgsLogisticRegression); + var result = context.Auto() - .CreateBinaryClassificationExperiment(1) + .CreateBinaryClassificationExperiment(settings) .Execute(dataTrainTest.TrainSet, dataTrainTest.TestSet, DatasetUtil.UciAdultLabel); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); @@ -72,7 +91,7 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() Assert.NotNull(result.BestRun.TrainerName); } - [LightGBMFact] + [Fact] public void AutoFit_UCI_Adult_CrossValidation_10_Test() { var context = new MLContext(1); @@ -80,15 +99,52 @@ public void AutoFit_UCI_Adult_CrossValidation_10_Test() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + settings.Trainers.Remove(BinaryClassificationTrainer.SdcaLogisticRegression); + settings.Trainers.Remove(BinaryClassificationTrainer.LbfgsLogisticRegression); var result = context.Auto() - .CreateBinaryClassificationExperiment(1) + .CreateBinaryClassificationExperiment(settings) .Execute(trainData, 10, DatasetUtil.UciAdultLabel); Assert.True(result.BestRun.Results.Select(x => x.ValidationMetrics.Accuracy).Min() > 0.70); Assert.NotNull(result.BestRun.Estimator); Assert.NotNull(result.BestRun.TrainerName); } - [LightGBMTheory] + [LightGBMFact] + public void SweepablePipeline_AutoFit_UCI_Adult_CrossValidation_10_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var dataPath = DatasetUtil.GetUciAdultDataset(); + var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); + var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); + var trainData = textLoader.Load(dataPath); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(trainData, columnInference.ColumnInformation) + .Append(context.Auto().BinaryClassification(labelColumnName: DatasetUtil.UciAdultLabel)); + + experiment.SetPipeline(pipeline) + .SetDataset(trainData, 5) + .SetBinaryClassificationMetric(BinaryClassificationMetric.Accuracy, DatasetUtil.UciAdultLabel) + .SetTrainingTimeInSeconds(10); + + var res = experiment.Run(); + res.Metric.Should().BeGreaterThan(0.5); + + } + + [Theory] [InlineData(true)] [InlineData(false)] public void AutoFitMultiTest(bool useNumberOfCVFolds) @@ -109,8 +165,17 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) // When setting numberOfCVFolds // The results object is a CrossValidationExperimentResults<> object uint numberOfCVFolds = 5; + var settings = new MulticlassExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm); + settings.Trainers.Remove(MulticlassClassificationTrainer.SdcaMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); var result = context.Auto() - .CreateMulticlassClassificationExperiment(5) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainData, numberOfCVFolds, DatasetUtil.TrivialMulticlassDatasetLabel); result.BestRun.Results.First().ValidationMetrics.MicroAccuracy.Should().BeGreaterThan(0.7); @@ -128,8 +193,17 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) int crossValRowCountThreshold = 15000; trainData = context.Data.TakeRows(trainData, crossValRowCountThreshold - 1); + var settings = new MulticlassExperimentSettings + { + MaxExperimentTimeInSeconds = 1, + }; + + settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm); + settings.Trainers.Remove(MulticlassClassificationTrainer.SdcaMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsMaximumEntropy); + settings.Trainers.Remove(MulticlassClassificationTrainer.LbfgsLogisticRegressionOva); var result = context.Auto() - .CreateMulticlassClassificationExperiment(10) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainData, DatasetUtil.TrivialMulticlassDatasetLabel); Assert.True(result.BestRun.ValidationMetrics.MicroAccuracy >= 0.7); @@ -198,7 +272,7 @@ public void AutoFitMultiClassification_Image() .CreateMulticlassClassificationExperiment(100) .Execute(trainData, columnInference.ColumnInformation); - Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.3, 0.9); + Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.1, 0.9); var scoredData = result.BestRun.Model.Transform(trainData); Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); } @@ -436,27 +510,31 @@ public void AutoFitWithPresplittedData() var dataFull = textLoader.Load(dataPath); var dataTrainTest = context.Data.TrainTestSplit(dataFull); var dataCV = context.Data.CrossValidationSplit(dataFull, numberOfFolds: 2); + var settings = new BinaryExperimentSettings + { + MaxExperimentTimeInSeconds = 10, + }; + + // remove fastForest because it doesn't calibrate score + // so column "probability" will be missing in the final result; + settings.Trainers.Remove(BinaryClassificationTrainer.FastForest); var modelFull = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(settings) .Execute(dataFull, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun .Model; - // AutoMLExperiment can't run on canceled context. - // Therefore, we need to create a new context after an experiment is completed. - context = new MLContext(1); var modelTrainTest = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(settings) .Execute(dataTrainTest.TrainSet, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun .Model; - context = new MLContext(1); var modelCV = context.Auto() - .CreateBinaryClassificationExperiment(10) + .CreateBinaryClassificationExperiment(settings) .Execute(dataCV.First().TrainSet, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }) .BestRun @@ -469,7 +547,6 @@ public void AutoFitWithPresplittedData() var resFull = model.Transform(dataFull); var resTrainTest = model.Transform(dataTrainTest.TrainSet); var resCV = model.Transform(dataCV.First().TrainSet); - Assert.Equal(31, resFull.Schema.Count); Assert.Equal(31, resTrainTest.Schema.Count); Assert.Equal(31, resCV.Schema.Count); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 5e343a3677..246ab5f66b 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -4,12 +4,14 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Data.Analysis; +using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Xunit; @@ -27,15 +29,16 @@ public AutoMLExperimentTests(ITestOutputHelper output) : base(output) public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_and_no_trial_completed_Async() { var context = new MLContext(1); - var pipeline = context.Transforms.Concatenate("Features", "Features") - .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(context, 5); var experiment = context.Auto().CreateExperiment(); - experiment.SetPipeline(pipeline) - .SetDataset(GetDummyData(), 10) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") - .SetTrainingTimeInSeconds(1) - .SetTrialRunner(dummyTrainer); + + experiment.SetTrainingTimeInSeconds(1) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 5, channel); + }) + .SetTuner(); var cts = new CancellationTokenSource(); @@ -56,16 +59,17 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async() { var context = new MLContext(1); - var pipeline = context.Transforms.Concatenate("Features", "Features") - .Append(context.Auto().Regression()); - - var dummyTrainer = new DummyTrialRunner(context, 1); + var stopWatch = new Stopwatch(); + stopWatch.Start(); var experiment = context.Auto().CreateExperiment(); - experiment.SetPipeline(pipeline) - .SetDataset(GetDummyData(), 10) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") - .SetTrainingTimeInSeconds(100) - .SetTrialRunner(dummyTrainer); + experiment.SetTrainingTimeInSeconds(10) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 1, channel); + }) + .SetTuner(); var cts = new CancellationTokenSource(); @@ -76,8 +80,11 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled cts.CancelAfter(100); } }; - var res = await experiment.RunAsync(cts.Token); + + stopWatch.Stop(); + stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000); + cts.IsCancellationRequested.Should().BeTrue(); res.Metric.Should().BeGreaterThan(0); } @@ -85,16 +92,16 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() { var context = new MLContext(1); - var pipeline = context.Transforms.Concatenate("Features", "Features") - .Append(context.Auto().Regression()); - var dummyTrainer = new DummyTrialRunner(context, 1); var experiment = context.Auto().CreateExperiment(); - experiment.SetPipeline(pipeline) - .SetDataset(GetDummyData(), 10) - .SetEvaluateMetric(RegressionMetric.RootMeanSquaredError, "Label") - .SetTrainingTimeInSeconds(5) - .SetTrialRunner(dummyTrainer); + experiment.SetTrainingTimeInSeconds(5) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 1, channel); + }) + .SetTuner(); var cts = new CancellationTokenSource(); cts.CancelAfter(10 * 1000); @@ -104,41 +111,189 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() cts.IsCancellationRequested.Should().BeFalse(); } - private IDataView GetDummyData() + [Fact] + public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() { - var x = Enumerable.Range(-10000, 10000).Select(value => value * 1f).ToArray(); - var y = x.Select(value => value * value); + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetUciAdultDataView(); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) + .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false, useLbfgs: false)); - var df = new DataFrame(); - df["Features"] = DataFrameColumn.Create("Features", x); - df["Label"] = DataFrameColumn.Create("Label", y); + experiment.SetDataset(context.Data.TrainTestSplit(data)) + .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(1); - return df; + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); } - class DummyTrialRunner : ITrialRunner + [Fact] + public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() { - private readonly int _finishAfterNSeconds; - private readonly MLContext _context; + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetUciAdultDataView(); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) + .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdca: false, useLbfgs: false)); - public DummyTrialRunner(MLContext context, int finishAfterNSeconds) + experiment.SetDataset(data, 5) + .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } + + [Fact] + public async Task AutoMLExperiment_Iris_CV_5_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => { - _finishAfterNSeconds = finishAfterNSeconds; - _context = context; - } + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetIrisDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = "Label"; + var pipeline = context.Auto().Featurizer(data, excludeColumns: new[] { label }) + .Append(context.Transforms.Conversion.MapValueToKey(label, label)) + .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(data, 5) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); - public TrialResult Run(TrialSettings settings, IServiceProvider provider = null) + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } + + [Fact] + public async Task AutoMLExperiment_Iris_Train_Test_Split_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => { - Task.Delay(_finishAfterNSeconds * 1000).Wait(); - settings.ExperimentSettings.CancellationToken.ThrowIfCancellationRequested(); + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var data = DatasetUtil.GetIrisDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = "Label"; + var pipeline = context.Auto().Featurizer(data, excludeColumns: new[] { label }) + .Append(context.Transforms.Conversion.MapValueToKey(label, label)) + .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(context.Data.TrainTestSplit(data)) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.8); + } - return new TrialResult + [Fact] + public async Task AutoMLExperiment_Taxi_Fare_Train_Test_Split_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) { - TrialSettings = settings, - DurationInMilliseconds = _finishAfterNSeconds * 1000, - Metric = 1.000 + 0.01 * settings.TrialId, - }; - } + this.Output.WriteLine(e.RawMessage); + } + }; + var train = DatasetUtil.GetTaxiFareTrainDataView(); + var test = DatasetUtil.GetTaxiFareTestDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = DatasetUtil.TaxiFareLabel; + var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label }) + .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(train, test) + .SetRegressionMetric(RegressionMetric.RSquared, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(50); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.5); + } + + [Fact] + public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var train = DatasetUtil.GetTaxiFareTrainDataView(); + var experiment = context.Auto().CreateExperiment(); + var label = DatasetUtil.TaxiFareLabel; + var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label }) + .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgs: false)); + + experiment.SetDataset(train, 5) + .SetRegressionMetric(RegressionMetric.RSquared, label) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(50); + + var result = await experiment.RunAsync(); + result.Metric.Should().BeGreaterThan(0.5); + } + } + + class DummyTrialRunner : ITrialRunner + { + private readonly int _finishAfterNSeconds; + private readonly CancellationToken _ct; + private readonly IChannel _logger; + + public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) + { + _finishAfterNSeconds = finishAfterNSeconds; + _ct = automlSettings.CancellationToken; + _logger = logger; + } + + public TrialResult Run(TrialSettings settings) + { + _logger.Info("Update Running Trial"); + Task.Delay(_finishAfterNSeconds * 1000).Wait(_ct); + _ct.ThrowIfCancellationRequested(); + _logger.Info("Update Completed Trial"); + return new TrialResult + { + TrialSettings = settings, + DurationInMilliseconds = _finishAfterNSeconds * 1000, + Metric = 1.000 + 0.01 * settings.TrialId, + }; } } } diff --git a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs index 9ccf051cf6..7eeead3bc9 100644 --- a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs +++ b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs @@ -16,24 +16,24 @@ namespace Microsoft.ML.AutoML.Test internal static class DatasetUtil { public const string UciAdultLabel = DefaultColumnNames.Label; + public const string TaxiFareLabel = "fare_amount"; public const string TrivialMulticlassDatasetLabel = "Target"; public const string MlNetGeneratedRegressionLabel = "target"; public const string NewspaperChurnLabel = "Subscriber"; - public const string TaxiFareLabel = "fare_amount"; public const int IrisDatasetLabelColIndex = 0; public static string TrivialMulticlassDatasetPath = Path.Combine("TestData", "TrivialMulticlassDataset.txt"); private static IDataView _uciAdultDataView; - private static IDataView _irisDataView; - - private static IDataView _newspaperChurnDataView; - private static IDataView _taxiFareTrainDataView; private static IDataView _taxiFareTestDataView; + private static IDataView _irisDataView; + + private static IDataView _newspaperChurnDataView; + public static string GetUciAdultDataset() => GetDataPath("adult.tiny.with-schema.txt"); public static string GetMlNetGeneratedRegressionDataset() => GetDataPath("generated_regression_dataset.csv"); @@ -100,6 +100,15 @@ public static IDataView GetNewspaperChurnDataView() return _newspaperChurnDataView; } + public static IDataView GetCreditApprovalDataView() + { + var context = new MLContext(1); + var file = GetDataPath(@"creditapproval_train.csv"); + var columnInferenceResult = context.Auto().InferColumns(file, "A16"); + var textLoader = context.Data.CreateTextLoader(columnInferenceResult.TextLoaderOptions); + return textLoader.Load(file); + } + public static IDataView GetTaxiFareTestDataView() { if (_taxiFareTestDataView == null) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index db6c5fe8d3..0afa1a1ea7 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -43,53 +44,41 @@ public SweepableExtensionTest(ITestOutputHelper output) } [Fact] - public void CreateSweepableEstimatorPipelineFromIEstimatorTest() + public void CreateSweepablePipelineFromIEstimatorTest() { var context = new MLContext(); var estimator = context.Transforms.Concatenate("output", "input"); var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); - pipeline.ToString().Should().Be("Unknown=>FastForestBinary"); + pipeline.Should().BeOfType(); } [Fact] - public void AppendIEstimatorToSweepabeEstimatorPipelineTest() - { - var context = new MLContext(); - var estimator = context.Transforms.Concatenate("output", "input"); - var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); - pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); - - pipeline.ToString().Should().Be("Unknown=>FastForestBinary=>Unknown"); - } - - [Fact] - public void CreateSweepableEstimatorPipelineFromSweepableEstimatorTest() + public void CreateSweepablePipelineFromSweepableEstimatorTest() { var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); var pipeline = estimator.Append(estimator); - pipeline.ToString().Should().Be("FastForestBinary=>FastForestBinary"); + pipeline.Should().BeOfType(); } [Fact] - public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorTest() + public void CreateSweepablePipelineFromSweepableEstimatorAndIEstimatorTest() { var context = new MLContext(); var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); var pipeline = estimator.Append(context.Transforms.Concatenate("output", "input")); - pipeline.ToString().Should().Be("FastForestBinary=>Unknown"); - + pipeline.Should().BeOfType(); } [Fact] [UseReporter(typeof(DiffReporter))] [UseApprovalSubdirectory("ApprovalTests")] - public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() + public void CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(context.Auto().BinaryClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); @@ -99,10 +88,10 @@ public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() + public void CreateSweepablePipelineFromIEstimatorAndMultiClassifiers() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(context.Auto().MultiClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); @@ -112,10 +101,10 @@ public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromIEstimatorAndRegressors() + public void CreateSweepablePipelineFromIEstimatorAndRegressors() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(context.Auto().MultiClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); @@ -125,7 +114,7 @@ public void CreateMultiModelPipelineFromIEstimatorAndRegressors() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() + public void CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers() { var context = new MLContext(); var pipeline = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()) @@ -138,15 +127,70 @@ public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() [Fact] [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] - public void CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers() + public void CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers() { var context = new MLContext(); - var pipeline = context.Transforms.Concatenate("output", "input") + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") .Append(SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption())) .Append(context.Auto().MultiClassification()); var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); Approvals.Verify(json); } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray() + { + var context = new MLContext(); + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") + .Append(SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption())) + .Append(context.Auto().MultiClassification().Estimators.Select(kv => kv.Value).ToArray()); + + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray() + { + var context = new MLContext(); + SweepablePipeline pipeline = SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption()) + .Append(context.Auto().MultiClassification().Estimators.Select(kv => kv.Value).ToArray()); + + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray() + { + var context = new MLContext(); + SweepablePipeline pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().MultiClassification().Estimators.Select(kv => kv.Value).ToArray()); + + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void AppendIEstimatorToSweepabePipelineTest() + { + var context = new MLContext(); + var estimator = context.Transforms.Concatenate("output", "input"); + var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()), SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); + + pipeline.Should().BeOfType(); + var json = JsonSerializer.Serialize(pipeline, _jsonSerializerOptions); + Approvals.Verify(json); + } } } diff --git a/test/data/creditapproval_train.csv b/test/data/creditapproval_train.csv new file mode 100644 index 0000000000..12c7aabba2 --- /dev/null +++ b/test/data/creditapproval_train.csv @@ -0,0 +1,553 @@ +A1,A2,A3,A4,A5,A6,A7,A8,A9,A10,A11,A12,A13,A14,A15,A16 +a,?,1.5,u,g,ff,ff,0.0,f,t,2,t,g,00200,105,- +a,46.00,4.0,u,g,j,j,0.0,t,f,0,f,g,00100,960,+ +b,20.00,0.0,u,g,d,v,0.5,f,f,0,f,g,00144,0,- +b,47.33,6.5,u,g,c,v,1.0,f,f,0,t,g,00000,228,- +b,19.17,0.0,y,p,m,bb,0.0,f,f,0,t,s,00500,1,+ +b,24.33,6.625,y,p,d,v,5.5,t,f,0,t,s,00100,0,+ +b,21.58,0.79,y,p,cc,v,0.665,f,f,0,f,g,00160,0,- +b,25.00,12.5,u,g,aa,v,3.0,t,f,0,t,s,00020,0,+ +b,21.92,0.5,u,g,c,v,0.125,f,f,0,f,g,00360,0,- +b,20.00,11.045,u,g,c,v,2.0,f,f,0,t,g,00136,0,- +a,23.50,1.5,u,g,w,v,0.875,f,f,0,t,g,00160,0,- +a,40.83,10.0,u,g,q,h,1.75,t,f,0,f,g,00029,837,+ +b,22.67,0.165,u,g,c,j,2.25,f,f,0,t,s,00000,0,+ +b,21.67,1.165,y,p,k,v,2.5,t,t,1,f,g,00180,20,- +a,30.58,10.665,u,g,q,h,0.085,f,t,12,t,g,00129,3,- +b,20.17,9.25,u,g,c,v,1.665,t,t,3,t,g,00040,28,+ +b,27.83,1.0,y,p,d,h,3.0,f,f,0,f,g,00176,537,- +a,16.08,0.335,u,g,ff,ff,0.0,f,t,1,f,g,00160,126,- +b,37.33,2.665,u,g,cc,v,0.165,f,f,0,t,g,00000,501,- +b,29.83,2.04,y,p,x,h,0.04,f,f,0,f,g,00128,1,- +a,16.00,0.165,u,g,aa,v,1.0,f,t,2,t,g,00320,1,- +b,34.75,2.5,u,g,cc,bb,0.5,f,f,0,f,g,00348,0,- +a,21.75,1.75,y,p,j,j,0.0,f,f,0,f,g,00160,0,- +b,21.92,0.54,y,p,x,v,0.04,t,t,1,t,g,00840,59,+ +b,23.92,0.585,y,p,cc,h,0.125,f,f,0,f,g,00240,1,- +?,40.83,3.5,u,g,i,bb,0.5,f,f,0,f,s,01160,0,- +b,40.25,21.5,u,g,e,z,20.0,t,t,11,f,g,00000,1200,+ +b,34.25,3.0,u,g,cc,h,7.415,t,f,0,t,g,00000,0,+ +a,22.50,0.415,u,g,i,v,0.335,f,f,0,t,s,00144,0,- +b,27.67,0.75,u,g,q,h,0.165,f,f,0,t,g,00220,251,- +b,36.25,5.0,u,g,c,bb,2.5,t,t,6,f,g,00000,367,+ +b,?,4.0,u,g,x,v,5.0,t,t,3,t,g,00290,2279,+ +b,39.92,0.54,y,p,aa,v,0.5,t,t,3,f,g,00200,1000,+ +a,19.75,0.75,u,g,c,v,0.795,t,t,5,t,g,00140,5,- +b,38.67,0.21,u,g,k,v,0.085,t,f,0,t,g,00280,0,+ +a,20.67,1.835,u,g,q,v,2.085,t,t,5,f,g,00220,2503,+ +b,60.08,14.5,u,g,ff,ff,18.0,t,t,15,t,g,00000,1000,+ +b,35.25,3.165,u,g,x,h,3.75,t,f,0,t,g,00680,0,- +b,18.50,2.0,u,g,i,v,1.5,t,t,2,f,g,00120,300,+ +a,47.42,8.0,u,g,e,bb,6.5,t,t,6,f,g,00375,51100,+ +b,21.83,11.0,u,g,x,v,0.29,t,t,6,f,g,00121,0,+ +a,38.58,5.0,u,g,cc,v,13.5,t,f,0,t,g,00980,0,- +b,30.75,1.585,u,g,d,v,0.585,f,f,0,t,s,00000,0,- +b,43.08,0.375,y,p,c,v,0.375,t,t,8,t,g,00300,162,+ +b,34.17,2.75,u,g,i,bb,2.5,f,f,0,t,g,00232,200,- +b,64.08,20.0,u,g,x,h,17.5,t,t,9,t,g,00000,1000,+ +a,38.92,1.665,u,g,aa,v,0.25,f,f,0,f,g,00000,390,- +b,34.00,5.085,y,p,i,bb,1.085,f,f,0,t,g,00480,0,- +b,34.08,6.5,u,g,aa,v,0.125,t,f,0,t,g,00443,0,- +b,22.75,11.5,u,g,i,v,0.415,f,f,0,f,g,00000,0,- +a,18.17,10.0,y,p,q,h,0.165,f,f,0,f,g,00340,0,- +b,62.50,12.75,y,p,c,h,5.0,t,f,0,f,g,00112,0,- +b,30.33,0.5,u,g,d,h,0.085,f,f,0,t,s,00252,0,- +b,22.08,0.83,u,g,c,h,2.165,f,f,0,t,g,00128,0,+ +a,20.42,0.835,u,g,q,v,1.585,t,t,1,f,g,00000,0,+ +a,22.67,7.0,u,g,c,v,0.165,f,f,0,f,g,00160,0,- +b,30.17,0.5,u,g,c,v,1.75,t,t,11,f,g,00032,540,+ +b,26.92,2.25,u,g,i,bb,0.5,f,f,0,t,g,00640,4000,- +b,48.33,12.0,u,g,m,v,16.0,t,f,0,f,s,00110,0,+ +a,15.75,0.375,u,g,c,v,1.0,f,f,0,f,g,00120,18,- +b,42.00,9.79,u,g,x,h,7.96,t,t,8,f,g,00000,0,+ +a,50.08,12.54,u,g,aa,v,2.29,t,t,3,t,g,00156,0,+ +b,20.42,7.0,u,g,c,v,1.625,t,t,3,f,g,00200,1391,+ +a,19.17,0.585,y,p,aa,v,0.585,t,f,0,t,g,00160,0,- +b,16.17,0.04,u,g,c,v,0.04,f,f,0,f,g,00000,0,+ +a,35.75,0.915,u,g,aa,v,0.75,t,t,4,f,g,00000,1583,+ +b,39.33,5.875,u,g,cc,h,10.0,t,t,14,t,g,00399,0,+ +b,23.33,11.625,y,p,w,v,0.835,t,f,0,t,g,00160,300,+ +b,48.58,6.5,u,g,q,h,6.0,t,f,0,t,g,00350,0,+ +a,24.83,4.5,u,g,w,v,1.0,f,f,0,t,g,00360,6,- +a,22.50,8.46,y,p,x,v,2.46,f,f,0,f,g,00164,0,+ +a,24.50,0.5,u,g,q,h,1.5,t,f,0,f,g,00280,824,+ +b,36.50,4.25,u,g,q,v,3.5,f,f,0,f,g,00454,50,- +a,27.67,1.5,u,g,m,v,2.0,t,f,0,f,s,00368,0,- +b,23.42,1.0,u,g,c,v,0.5,f,f,0,t,s,00280,0,- +b,21.08,10.085,y,p,e,h,1.25,f,f,0,f,g,00260,0,- +b,21.25,1.5,u,g,w,v,1.5,f,f,0,f,g,00150,8,+ +b,25.75,0.75,u,g,c,bb,0.25,t,f,0,f,g,00349,23,+ +b,39.92,6.21,u,g,q,v,0.04,t,t,1,f,g,00200,300,+ +b,25.83,12.835,u,g,cc,v,0.5,f,f,0,f,g,00000,2,- +a,19.67,0.21,u,g,q,h,0.29,t,t,11,f,g,00080,99,+ +a,22.42,11.25,y,p,x,h,0.75,t,t,4,f,g,00000,321,+ +b,19.42,7.25,u,g,m,v,0.04,f,t,1,f,g,00100,1,- +b,22.67,0.75,u,g,i,v,1.585,f,t,1,t,g,00400,9,- +b,20.42,1.835,u,g,c,v,2.25,t,t,1,f,g,00100,150,+ +a,30.25,5.5,u,g,k,v,5.5,f,f,0,t,s,00100,0,- +b,56.75,12.25,u,g,m,v,1.25,t,t,4,t,g,00200,0,+ +a,30.50,6.5,u,g,c,bb,4.0,t,t,7,t,g,00000,3065,+ +b,29.25,14.79,u,g,aa,v,5.04,t,t,5,t,g,00168,0,+ +b,35.17,25.125,u,g,x,h,1.625,t,t,1,t,g,00515,500,+ +a,20.50,11.835,u,g,c,h,6.0,t,f,0,f,g,00340,0,+ +a,18.83,9.54,u,g,aa,v,0.085,t,f,0,f,g,00100,0,+ +b,56.42,28.0,y,p,c,v,28.5,t,t,40,f,g,00000,15,+ +a,57.08,0.335,u,g,i,bb,1.0,t,f,0,t,g,00252,2197,- +b,32.33,7.5,u,g,e,bb,1.585,t,f,0,t,s,00420,0,- +a,20.67,3.0,u,g,q,v,0.165,t,t,3,f,g,00100,6,+ +a,60.58,16.5,u,g,q,v,11.0,t,f,0,t,g,00021,10561,+ +b,41.58,1.75,u,g,k,v,0.21,t,f,0,f,g,00160,0,- +b,41.17,1.335,u,g,d,v,0.165,f,f,0,f,g,00168,0,- +a,21.08,5.0,y,p,ff,ff,0.0,f,f,0,f,g,00000,0,- +a,69.50,6.0,u,g,ff,ff,0.0,f,f,0,f,s,00000,0,- +a,18.08,0.375,l,gg,cc,ff,10.0,f,f,0,t,s,00300,0,+ +?,24.50,12.75,u,g,c,bb,4.75,t,t,2,f,g,00073,444,+ +a,24.92,1.25,u,g,ff,ff,0.0,t,f,0,f,g,00080,0,- +b,34.17,1.54,u,g,cc,v,1.54,t,t,1,t,g,00520,50000,+ +b,45.17,1.5,u,g,c,v,2.5,t,f,0,t,g,00140,0,- +b,23.92,0.665,u,g,c,v,0.165,f,f,0,f,g,00100,0,+ +b,57.83,7.04,u,g,m,v,14.0,t,t,6,t,g,00360,1332,+ +b,50.75,0.585,u,g,ff,ff,0.0,f,f,0,f,g,00145,0,- +b,54.58,9.415,u,g,ff,ff,14.415,t,t,11,t,g,00030,300,+ +b,32.25,14.0,y,p,ff,ff,0.0,f,t,2,f,g,00160,1,- +b,41.75,0.96,u,g,x,v,2.5,t,f,0,f,g,00510,600,+ +b,33.17,1.04,u,g,r,h,6.5,t,f,0,t,g,00164,31285,+ +b,35.25,16.5,y,p,c,v,4.0,t,f,0,f,g,00080,0,- +a,21.67,11.5,y,p,j,j,0.0,t,t,11,t,g,00000,0,+ +b,69.17,9.0,u,g,ff,ff,4.0,f,t,1,f,g,00070,6,- +a,47.42,3.0,u,g,x,v,13.875,t,t,2,t,g,00519,1704,+ +b,22.58,10.04,u,g,x,v,0.04,t,t,9,f,g,00060,396,+ +b,23.25,12.625,u,g,c,v,0.125,f,t,2,f,g,00000,5552,- +b,32.83,2.5,u,g,cc,h,2.75,t,t,6,f,g,00160,2072,+ +b,42.75,4.085,u,g,aa,v,0.04,f,f,0,f,g,00108,100,- +b,23.08,2.5,u,g,ff,ff,0.085,f,f,0,t,g,00100,4208,- +b,23.08,11.5,u,g,i,v,3.5,t,t,9,f,g,00056,742,+ +a,22.67,0.75,u,g,c,v,2.0,f,t,2,t,g,00200,394,- +b,51.83,2.04,y,p,ff,ff,1.5,f,f,0,f,g,00120,1,- +b,?,0.375,u,g,d,v,0.875,t,f,0,t,s,00928,0,- +b,36.67,2.0,u,g,i,v,0.25,f,f,0,t,g,00221,0,- +b,27.58,3.25,y,p,q,h,5.085,f,t,2,t,g,00369,1,- +b,29.58,4.5,u,g,w,v,7.5,t,t,2,t,g,00330,0,+ +b,28.92,15.0,u,g,c,h,5.335,t,t,11,f,g,00000,2283,+ +b,40.92,2.25,y,p,x,h,10.0,t,f,0,t,g,00176,0,- +b,35.58,0.75,u,g,k,v,1.5,f,f,0,t,g,00231,0,- +b,23.17,0.0,u,g,cc,v,0.085,t,f,0,f,g,00000,0,+ +a,25.25,12.5,u,g,d,v,1.0,f,f,0,t,g,00180,1062,- +a,24.75,3.0,u,g,q,h,1.835,t,t,19,f,g,00000,500,+ +a,41.17,6.5,u,g,q,v,0.5,t,t,3,t,g,00145,0,+ +b,28.00,2.0,u,g,k,h,4.165,t,t,2,t,g,00181,0,+ +a,28.17,0.375,u,g,q,v,0.585,t,t,4,f,g,00080,0,+ +b,24.58,13.5,y,p,ff,ff,0.0,f,f,0,f,g,?,0,- +b,29.17,3.5,u,g,w,v,3.5,t,t,3,t,g,00329,0,- +b,39.83,0.5,u,g,m,v,0.25,t,f,0,f,s,00288,0,- +b,26.00,1.0,u,g,q,v,1.75,t,f,0,t,g,00280,0,+ +b,53.92,9.625,u,g,e,v,8.665,t,t,5,f,g,00000,0,+ +b,34.75,15.0,u,g,r,n,5.375,t,t,9,t,g,00000,134,+ +b,16.50,0.125,u,g,c,v,0.165,f,f,0,f,g,00132,0,- +b,25.42,0.54,u,g,w,v,0.165,f,t,1,f,g,00272,444,- +b,16.25,0.0,y,p,aa,v,0.25,f,f,0,f,g,00060,0,- +b,31.83,2.5,u,g,aa,v,7.5,t,f,0,t,g,00523,0,- +b,21.83,0.25,u,g,d,h,0.665,t,f,0,t,g,00000,0,+ +b,48.08,3.75,u,g,i,bb,1.0,f,f,0,f,g,00100,2,- +b,24.50,13.335,y,p,aa,v,0.04,f,f,0,t,g,00120,475,- +b,33.67,2.165,u,g,c,v,1.5,f,f,0,f,p,00120,0,- +b,?,0.04,y,p,d,v,4.25,f,f,0,t,g,00460,0,- +b,31.67,16.165,u,g,d,v,3.0,t,t,9,f,g,00250,730,+ +a,27.25,0.29,u,g,m,h,0.125,f,t,1,t,g,00272,108,- +b,39.00,5.0,u,g,cc,v,3.5,t,t,10,t,g,00000,0,+ +b,42.08,1.04,u,g,w,v,5.0,t,t,6,t,g,00500,10000,+ +a,21.25,2.335,u,g,i,bb,0.5,t,t,4,f,s,00080,0,+ +a,22.67,0.79,u,g,i,v,0.085,f,f,0,f,g,00144,0,- +a,30.42,1.375,u,g,w,h,0.04,f,t,3,f,g,00000,33,- +a,47.25,0.75,u,g,q,h,2.75,t,t,1,f,g,00333,892,+ +b,22.92,3.165,y,p,c,v,0.165,f,f,0,f,g,00160,1058,- +b,35.00,2.5,u,g,i,v,1.0,f,f,0,t,g,00210,0,- +b,16.92,0.335,y,p,k,v,0.29,f,f,0,f,s,00200,0,- +a,33.25,3.0,y,p,aa,v,2.0,f,f,0,f,g,00180,0,- +b,20.67,0.835,y,p,c,v,2.0,f,f,0,t,s,00240,0,- +b,24.58,1.25,u,g,c,v,0.25,f,f,0,f,g,00110,0,- +b,20.50,2.415,u,g,c,v,2.0,t,t,11,t,g,00200,3000,+ +a,19.50,0.165,u,g,q,v,0.04,f,f,0,t,g,00380,0,- +b,48.17,3.5,u,g,aa,v,3.5,t,f,0,f,s,00230,0,+ +a,19.58,0.665,u,g,w,v,1.665,f,f,0,f,g,00220,5,- +b,15.17,7.0,u,g,e,v,1.0,f,f,0,f,g,00600,0,- +b,22.67,10.5,u,g,q,h,1.335,t,f,0,f,g,00100,0,+ +b,24.08,9.0,u,g,aa,v,0.25,f,f,0,t,g,00000,0,- +b,32.00,1.75,y,p,e,h,0.04,t,f,0,t,g,00393,0,+ +b,43.17,5.0,u,g,i,bb,2.25,f,f,0,t,g,00141,0,- +b,20.08,0.25,u,g,q,v,0.125,f,f,0,f,g,00200,0,- +b,21.83,1.54,u,g,k,v,0.085,f,f,0,t,g,00356,0,- +a,39.08,4.0,u,g,c,v,3.0,f,f,0,f,g,00480,0,- +b,18.58,5.71,u,g,d,v,0.54,f,f,0,f,g,00120,0,- +a,17.33,9.5,u,g,aa,v,1.75,f,t,10,t,g,00000,10,- +b,27.83,4.0,y,p,i,h,5.75,t,t,2,t,g,00075,0,- +b,20.67,1.25,y,p,c,h,1.375,t,t,3,t,g,00140,210,- +b,29.58,4.75,u,g,m,v,2.0,f,t,1,t,g,00460,68,- +b,35.17,2.5,u,g,k,v,4.5,t,t,7,f,g,00150,1270,+ +b,32.75,2.335,u,g,d,h,5.75,f,f,0,t,g,00292,0,- +b,?,5.0,y,p,aa,v,8.5,t,f,0,f,g,00000,0,- +a,22.92,11.585,u,g,cc,v,0.04,t,f,0,f,g,00080,1349,+ +a,49.00,1.5,u,g,j,j,0.0,t,f,0,t,g,00100,27,- +b,22.83,3.0,u,g,m,v,1.29,t,t,1,f,g,00260,800,+ +a,33.08,1.625,u,g,d,v,0.54,f,f,0,t,g,00000,0,- +b,?,10.5,u,g,x,v,6.5,t,f,0,f,g,00000,0,+ +b,39.42,1.71,y,p,m,v,0.165,f,f,0,f,s,00400,0,- +b,80.25,5.5,u,g,?,?,0.54,t,f,0,f,g,00000,340,- +b,36.33,2.125,y,p,w,v,0.085,t,t,1,f,g,00050,1187,+ +a,31.25,3.75,u,g,cc,h,0.625,t,t,9,t,g,00181,0,+ +b,57.42,8.5,u,g,e,h,7.0,t,t,3,f,g,00000,0,+ +b,39.58,13.915,u,g,w,v,8.625,t,t,6,t,g,00070,0,+ +b,29.67,1.415,u,g,w,h,0.75,t,t,1,f,g,00240,100,+ +b,44.25,0.5,u,g,m,v,10.75,t,f,0,f,s,00400,0,- +a,32.33,0.54,u,g,cc,v,0.04,t,f,0,f,g,00440,11177,+ +b,39.50,4.25,u,g,c,bb,6.5,t,t,16,f,g,00117,1210,+ +?,42.25,1.75,y,p,?,?,0.0,f,f,0,t,g,00150,1,- +a,36.00,1.0,u,g,c,v,2.0,t,t,11,f,g,00000,456,+ +b,29.92,1.835,u,g,c,h,4.335,t,f,0,f,g,00260,200,+ +a,37.33,2.5,u,g,i,h,0.21,f,f,0,f,g,00260,246,- +a,22.83,2.29,u,g,q,h,2.29,t,t,7,t,g,00140,2384,+ +a,18.42,9.25,u,g,q,v,1.21,t,t,4,f,g,00060,540,+ +b,17.08,0.085,y,p,c,v,0.04,f,f,0,f,g,00140,722,- +b,22.50,11.5,y,p,m,v,1.5,f,f,0,t,g,00000,4000,- +b,43.25,25.21,u,g,q,h,0.21,t,t,1,f,g,00760,90,- +b,17.08,0.25,u,g,q,v,0.335,f,t,4,f,g,00160,8,- +b,26.67,1.75,y,p,c,v,1.0,t,t,5,t,g,00160,5777,+ +a,26.08,8.665,u,g,aa,v,1.415,t,f,0,f,g,00160,150,+ +b,20.00,7.0,u,g,c,v,0.5,f,f,0,f,g,00000,0,- +b,25.67,3.25,u,g,c,h,2.29,f,t,1,t,g,00416,21,- +a,?,3.5,u,g,d,v,3.0,t,f,0,t,g,00300,0,- +a,27.33,1.665,u,g,ff,ff,0.0,f,f,0,f,g,00340,1,- +b,27.83,1.5,u,g,w,v,2.25,f,t,1,t,g,00100,3,- +?,29.75,0.665,u,g,w,v,0.25,f,f,0,t,g,00300,0,- +b,21.50,9.75,u,g,c,v,0.25,t,f,0,f,g,00140,0,- +a,27.42,14.5,u,g,x,h,3.085,t,t,1,f,g,00120,11,+ +b,36.67,4.415,y,p,k,v,0.25,t,t,10,t,g,00320,0,+ +a,20.83,0.5,y,p,e,dd,1.0,f,f,0,f,g,00260,0,- +a,58.42,21.0,u,g,i,bb,10.0,t,t,13,f,g,00000,6700,+ +b,36.17,0.42,y,p,w,v,0.29,f,f,0,t,g,00309,2,- +b,32.33,2.5,u,g,c,v,1.25,f,f,0,t,g,00280,0,- +b,19.00,1.75,y,p,c,v,2.335,f,f,0,t,g,00112,6,- +a,24.33,2.5,y,p,i,bb,4.5,f,f,0,f,g,00200,456,- +b,33.58,2.75,u,g,m,v,4.25,t,t,6,f,g,00204,0,+ +?,32.25,1.5,u,g,c,v,0.25,f,f,0,t,g,00372,122,- +b,30.17,1.085,y,p,c,v,0.04,f,f,0,f,g,00170,179,- +a,36.75,5.125,u,g,e,v,5.0,t,f,0,t,g,00000,4000,+ +a,27.67,2.04,u,g,w,v,0.25,f,f,0,t,g,00180,50,- +b,39.17,2.5,y,p,i,h,10.0,f,f,0,t,s,00200,0,- +b,42.50,4.915,y,p,w,v,3.165,t,f,0,t,g,00052,1442,+ +a,47.75,8.0,u,g,c,v,7.875,t,t,6,t,g,00000,1260,+ +a,57.58,2.0,u,g,ff,ff,6.5,f,t,1,f,g,00000,10,- +b,65.42,11.0,u,g,e,z,20.0,t,t,7,t,g,00022,0,+ +b,26.17,12.5,y,p,k,h,1.25,f,f,0,t,g,00000,17,- +a,41.58,1.04,u,g,aa,v,0.665,f,f,0,f,g,00240,237,- +b,76.75,22.29,u,g,e,z,12.75,t,t,1,t,g,00000,109,+ +b,27.25,0.625,u,g,aa,v,0.455,t,f,0,t,g,00200,0,- +a,33.25,2.5,y,p,c,v,2.5,f,f,0,t,g,00000,2,- +b,34.08,2.5,u,g,c,v,1.0,f,f,0,f,g,00460,16,- +b,39.17,1.625,u,g,c,v,1.5,t,t,10,f,g,00186,4700,+ +a,24.50,1.75,y,p,c,v,0.165,f,f,0,f,g,00132,0,- +a,20.75,10.25,u,g,q,v,0.71,t,t,2,t,g,00049,0,+ +b,33.17,3.165,y,p,x,v,3.165,t,t,3,t,g,00380,0,+ +a,15.83,7.625,u,g,q,v,0.125,f,t,1,t,g,00000,160,- +b,20.25,9.96,u,g,e,dd,0.0,t,f,0,f,g,00000,0,+ +b,24.75,0.54,u,g,m,v,1.0,f,f,0,t,g,00120,1,- +b,52.33,1.375,y,p,c,h,9.46,t,f,0,t,g,00200,100,- +b,18.83,0.0,u,g,q,v,0.665,f,f,0,f,g,00160,1,- +a,18.25,10.0,u,g,w,v,1.0,f,t,1,f,g,00120,1,- +b,22.17,12.125,u,g,c,v,3.335,f,t,2,t,g,00180,173,- +b,21.42,0.75,y,p,r,n,0.75,f,f,0,t,g,00132,2,- +a,22.42,5.665,u,g,q,v,2.585,t,t,7,f,g,00129,3257,+ +a,25.17,2.875,u,g,x,h,0.875,t,f,0,f,g,00360,0,+ +b,23.00,0.75,u,g,m,v,0.5,f,f,0,t,s,00320,0,- +b,23.08,2.5,u,g,c,v,1.085,t,t,11,t,g,00060,2184,+ +b,20.67,5.29,u,g,q,v,0.375,t,t,1,f,g,00160,0,- +b,34.83,2.5,y,p,w,v,3.0,f,f,0,f,s,00200,0,- +b,19.42,6.5,u,g,w,h,1.46,t,t,7,f,g,00080,2954,+ +b,40.92,0.835,u,g,ff,ff,0.0,t,f,0,f,g,00130,1,- +a,28.67,1.04,u,g,c,v,2.5,t,t,5,t,g,00300,1430,+ +b,26.17,0.25,u,g,i,bb,0.0,t,f,0,t,g,00000,0,+ +b,34.00,4.5,u,g,aa,v,1.0,t,f,0,t,g,00240,0,- +b,41.00,0.04,u,g,e,v,0.04,f,t,1,f,s,00560,0,+ +b,52.50,6.5,u,g,k,v,6.29,t,t,15,f,g,00000,11202,+ +b,34.92,5.0,u,g,x,h,7.5,t,t,6,t,g,00000,1000,+ +b,19.50,9.585,u,g,aa,v,0.79,f,f,0,f,g,00080,350,- +b,42.83,4.625,u,g,q,v,4.58,t,f,0,f,s,00000,0,+ +b,30.83,0.0,u,g,w,v,1.25,t,t,1,f,g,00202,0,+ +b,17.08,3.29,u,g,i,v,0.335,f,f,0,t,g,00140,2,- +b,33.17,1.0,u,g,x,v,0.75,t,t,7,t,g,00340,4071,+ +b,22.08,2.335,u,g,k,v,0.75,f,f,0,f,g,00180,0,- +b,23.58,1.79,u,g,c,v,0.54,f,f,0,t,g,00136,1,- +b,37.75,7.0,u,g,q,h,11.5,t,t,7,t,g,00300,5,- +b,19.67,0.375,u,g,q,v,2.0,t,t,2,t,g,00080,0,+ +b,23.50,2.75,u,g,ff,ff,4.5,f,f,0,f,g,00160,25,- +b,25.33,0.58,u,g,c,v,0.29,t,t,7,t,g,00096,5124,+ +b,30.67,2.5,u,g,cc,h,2.25,f,f,0,t,s,00340,0,- +b,19.58,0.585,u,g,ff,ff,0.0,f,t,3,f,g,00350,769,- +b,27.25,1.665,u,g,cc,h,5.085,t,t,9,f,g,00399,827,+ +b,19.67,10.0,y,p,k,h,0.835,t,f,0,t,g,00140,0,- +b,32.42,2.165,y,p,k,ff,0.0,f,f,0,f,g,00120,0,- +a,25.58,0.0,?,?,?,?,0.0,f,f,0,f,p,?,0,+ +a,29.58,1.75,y,p,k,v,1.25,f,f,0,t,g,00280,0,- +b,59.67,1.54,u,g,q,v,0.125,t,f,0,t,g,00260,0,+ +a,17.92,10.21,u,g,ff,ff,0.0,f,f,0,f,g,00000,50,- +a,41.00,2.04,y,p,q,h,0.125,t,t,23,t,g,00455,1236,+ +a,52.17,0.0,y,p,ff,ff,0.0,f,f,0,f,g,00000,0,- +b,28.25,5.04,y,p,c,bb,1.5,t,t,8,t,g,00144,7,+ +b,35.00,3.375,u,g,c,h,8.29,f,f,0,t,g,00000,0,- +a,20.33,10.0,u,g,c,h,1.0,t,t,4,f,g,00050,1465,+ +a,25.00,11.0,y,p,aa,v,4.5,t,f,0,f,g,00120,0,- +b,34.25,1.75,u,g,w,bb,0.25,t,f,0,t,g,00163,0,- +b,22.67,2.54,y,p,c,h,2.585,t,f,0,f,g,00000,0,+ +b,36.75,0.125,y,p,c,v,1.5,f,f,0,t,g,00232,113,+ +b,37.50,1.75,y,p,c,bb,0.25,t,f,0,t,g,00164,400,- +b,29.67,0.75,y,p,c,v,0.04,f,f,0,f,g,00240,0,- +b,29.42,1.25,u,g,c,h,0.25,f,t,2,t,g,00400,108,- +b,?,4.0,y,p,i,v,0.085,f,f,0,t,g,00411,0,- +b,20.17,8.17,u,g,aa,v,1.96,t,t,14,f,g,00060,158,+ +a,24.75,13.665,u,g,q,h,1.5,f,f,0,f,g,00280,1,- +b,38.92,1.75,u,g,k,v,0.5,f,f,0,t,g,00300,2,- +a,52.83,15.0,u,g,c,v,5.5,t,t,14,f,g,00000,2200,+ +a,23.50,9.0,u,g,q,v,8.5,t,t,5,t,g,00120,0,+ +b,44.25,11.0,y,p,d,v,1.5,t,f,0,f,s,00000,0,- +a,46.08,3.0,u,g,c,v,2.375,t,t,8,t,g,00396,4159,+ +b,48.75,8.5,u,g,c,h,12.5,t,t,9,f,g,00181,1655,+ +b,29.42,1.25,u,g,w,v,1.75,f,f,0,f,g,00200,0,- +b,26.67,14.585,u,g,i,bb,0.0,f,f,0,t,g,00178,0,- +b,19.50,0.29,u,g,k,v,0.29,f,f,0,f,g,00280,364,- +a,38.75,1.5,u,g,ff,ff,0.0,f,f,0,f,g,00076,0,- +b,18.17,2.46,u,g,c,n,0.96,f,t,2,t,g,00160,587,- +b,43.00,0.29,y,p,cc,h,1.75,t,t,8,f,g,00100,375,+ +a,56.50,16.0,u,g,j,ff,0.0,t,t,15,f,g,00000,247,+ +b,22.00,7.835,y,p,i,bb,0.165,f,f,0,t,g,?,0,- +a,19.17,8.585,u,g,cc,h,0.75,t,t,7,f,g,00096,0,+ +b,56.00,12.5,u,g,k,h,8.0,t,f,0,t,g,00024,2028,+ +a,32.17,1.46,u,g,w,v,1.085,t,t,16,f,g,00120,2079,+ +a,16.33,0.21,u,g,aa,v,0.125,f,f,0,f,g,00200,1,- +?,33.17,2.25,y,p,cc,v,3.5,f,f,0,t,g,00200,141,- +a,38.25,6.0,u,g,k,v,1.0,t,f,0,t,g,00000,0,+ +b,48.75,26.335,y,p,ff,ff,0.0,t,f,0,t,g,00000,0,- +a,15.83,0.585,u,g,c,h,1.5,t,t,2,f,g,00100,0,+ +a,44.33,0.0,u,g,c,v,2.5,t,f,0,f,g,00000,0,+ +b,37.50,1.125,y,p,d,v,1.5,f,f,0,t,g,00431,0,+ +b,43.17,2.25,u,g,i,bb,0.75,t,f,0,f,g,00560,0,- +b,16.00,3.125,u,g,w,v,0.085,f,t,1,f,g,00000,6,- +b,33.17,3.04,y,p,c,h,2.04,t,t,1,t,g,00180,18027,+ +b,16.33,2.75,u,g,aa,v,0.665,f,t,1,f,g,00080,21,- +b,49.17,2.29,u,g,ff,ff,0.29,f,f,0,f,g,00200,3,- +b,18.08,6.75,y,p,m,v,0.04,f,f,0,f,g,00140,0,- +b,32.75,1.5,u,g,cc,h,5.5,t,t,3,t,g,00000,0,+ +b,47.67,2.5,u,g,m,bb,2.5,t,t,12,t,g,00410,2510,+ +a,23.00,1.835,u,g,j,j,0.0,f,t,1,f,g,00200,53,- +b,29.50,0.58,u,g,w,v,0.29,f,t,1,f,g,00340,2803,- +a,33.75,0.75,u,g,k,bb,1.0,t,t,3,t,g,00212,0,- +b,26.83,0.54,u,g,k,ff,0.0,f,f,0,f,g,00100,0,- +a,49.83,13.585,u,g,k,h,8.5,t,f,0,t,g,00000,0,+ +a,26.17,2.0,u,g,j,j,0.0,f,f,0,t,g,00276,1,- +b,44.33,0.5,u,g,i,h,5.0,t,f,0,t,g,00320,0,+ +a,53.33,0.165,u,g,ff,ff,0.0,f,f,0,t,s,00062,27,- +a,17.58,9.0,u,g,aa,v,1.375,t,f,0,t,g,00000,0,+ +b,21.17,0.875,y,p,c,h,0.25,f,f,0,f,g,00280,204,- +a,23.25,5.875,u,g,q,v,3.17,t,t,10,f,g,00120,245,+ +b,31.00,2.085,u,g,c,v,0.085,f,f,0,f,g,00300,0,- +a,33.67,0.375,u,g,cc,v,0.375,f,f,0,f,g,00300,44,+ +b,18.17,10.25,u,g,c,h,1.085,f,f,0,f,g,00320,13,- +b,25.17,3.0,u,g,c,v,1.25,f,t,1,f,g,00000,22,- +b,22.25,9.0,u,g,aa,v,0.085,f,f,0,f,g,00000,0,- +b,21.33,10.5,u,g,c,v,3.0,t,f,0,t,g,00000,0,+ +a,27.58,3.0,u,g,m,v,2.79,f,t,1,t,g,00280,10,- +b,32.33,3.5,u,g,k,v,0.5,f,f,0,t,g,00232,0,- +b,27.00,0.75,u,g,c,h,4.25,t,t,3,t,g,00312,150,+ +b,28.75,1.5,y,p,c,v,1.5,t,f,0,t,g,00000,225,- +b,47.67,0.29,u,g,c,bb,15.0,t,t,20,f,g,00000,15000,+ +b,18.25,0.165,u,g,d,v,0.25,f,f,0,t,s,00280,0,- +b,33.58,0.335,y,p,cc,v,0.085,f,f,0,f,g,00180,0,- +b,28.25,5.125,u,g,x,v,4.75,t,t,2,f,g,00420,7,+ +b,31.25,1.125,u,g,ff,ff,0.0,f,t,1,f,g,00096,19,- +a,?,11.25,u,g,ff,ff,0.0,f,f,0,f,g,?,5200,- +b,19.17,4.0,y,p,i,v,1.0,f,f,0,t,g,00360,1000,- +b,32.08,4.0,u,g,m,v,2.5,t,f,0,t,g,00360,0,+ +b,54.33,6.75,u,g,c,h,2.625,t,t,11,t,g,00000,284,+ +b,59.50,2.75,u,g,w,v,1.75,t,t,5,t,g,00060,58,+ +b,39.17,1.71,u,g,x,v,0.125,t,t,5,t,g,00480,0,+ +b,41.33,0.0,u,g,c,bb,15.0,t,f,0,f,g,00000,0,+ +b,28.25,0.875,u,g,m,v,0.96,t,t,3,t,g,00396,0,+ +b,41.50,1.54,u,g,i,bb,3.5,f,f,0,f,g,00216,0,+ +a,21.75,11.75,u,g,c,v,0.25,f,f,0,t,g,00180,0,- +a,22.92,1.25,u,g,q,v,0.25,f,f,0,t,g,00120,809,- +b,27.83,1.54,u,g,w,v,3.75,t,t,5,t,g,00100,3,+ +b,17.25,3.0,u,g,k,v,0.04,f,f,0,t,g,00160,40,- +b,36.08,2.54,u,g,ff,ff,0.0,f,f,0,f,g,00000,1000,- +b,21.17,0.25,y,p,c,h,0.25,f,f,0,f,g,00280,204,- +a,23.75,0.71,u,g,w,v,0.25,f,t,1,t,g,00240,4,- +a,16.50,1.25,u,g,q,v,0.25,f,t,1,f,g,00108,98,- +a,18.83,4.415,y,p,c,h,3.0,t,f,0,f,g,00240,0,+ +b,17.58,10.0,u,g,w,h,0.165,f,t,1,f,g,00120,1,- +b,58.58,2.71,u,g,c,v,2.415,f,f,0,t,g,00320,0,- +b,23.25,4.0,u,g,c,bb,0.25,t,f,0,t,g,00160,0,+ +a,47.00,13.0,u,g,i,bb,5.165,t,t,9,t,g,00000,0,+ +b,42.75,3.0,u,g,i,bb,1.0,t,f,0,f,g,00000,200,- +a,20.42,10.5,y,p,x,h,0.0,f,f,0,t,g,00154,32,- +a,25.75,0.5,u,g,c,h,0.875,t,f,0,t,g,00491,0,+ +b,24.42,2.0,u,g,e,dd,0.165,f,t,2,f,g,00320,1300,- +b,22.08,11.0,u,g,cc,v,0.665,t,f,0,f,g,00100,0,+ +a,23.00,11.75,u,g,x,h,0.5,t,t,2,t,g,00300,551,+ +a,68.67,15.0,u,g,e,z,0.0,t,t,14,f,g,00000,3376,+ +b,21.33,7.5,u,g,aa,v,1.415,t,t,1,f,g,00080,9800,+ +a,50.25,0.835,u,g,aa,v,0.5,f,f,0,t,g,00240,117,- +?,28.17,0.585,u,g,aa,v,0.04,f,f,0,f,g,00260,1004,- +b,33.00,2.5,y,p,w,v,7.0,f,f,0,t,g,00280,0,- +b,27.00,1.5,y,p,w,v,0.375,t,f,0,t,g,00260,1065,+ +b,48.50,4.25,u,g,m,v,0.125,t,f,0,t,g,00225,0,+ +b,23.92,1.5,u,g,d,h,1.875,t,t,6,f,g,00200,327,+ +a,18.92,9.0,u,g,aa,v,0.75,t,t,2,f,g,00088,591,+ +b,34.50,4.04,y,p,i,bb,8.5,t,t,7,t,g,00195,0,+ +a,28.08,15.0,y,p,e,z,0.0,t,f,0,f,g,00000,13212,+ +b,34.17,5.25,u,g,w,v,0.085,f,f,0,t,g,00290,6,+ +b,51.83,3.0,y,p,ff,ff,1.5,f,f,0,f,g,00180,4,- +b,25.92,0.875,u,g,k,v,0.375,f,t,2,t,g,00174,3,- +b,21.50,11.5,u,g,i,v,0.5,t,f,0,t,g,00100,68,- +b,31.83,0.04,y,p,m,v,0.04,f,f,0,f,g,00000,0,- +b,21.00,3.0,y,p,d,v,1.085,t,t,8,t,g,00160,1,+ +b,30.17,6.5,u,g,cc,v,3.125,t,t,8,f,g,00330,1200,+ +b,25.50,0.375,u,g,m,v,0.25,t,t,3,f,g,00260,15108,+ +b,17.92,0.205,u,g,aa,v,0.04,f,f,0,f,g,00280,750,- +b,36.67,3.25,u,g,q,h,9.0,t,f,0,t,g,00102,639,+ +a,55.75,7.08,u,g,k,h,6.75,t,t,3,t,g,00100,50,- +b,25.08,1.71,u,g,x,v,1.665,t,t,1,t,g,00395,20,+ +a,31.75,3.0,y,p,j,j,0.0,f,f,0,f,g,00160,20,- +b,22.67,1.585,y,p,w,v,3.085,t,t,6,f,g,00080,0,+ +a,21.92,11.665,u,g,k,h,0.085,f,f,0,f,g,00320,5,- +b,42.83,1.25,u,g,m,v,13.875,f,t,1,t,g,00352,112,- +a,56.83,4.25,y,p,ff,ff,5.0,f,f,0,t,g,00000,4,- +b,22.17,2.25,u,g,i,v,0.125,f,f,0,f,g,00160,10,- +b,18.83,3.54,y,p,ff,ff,0.0,f,f,0,t,g,00180,1,- +b,37.33,6.5,u,g,m,h,4.25,t,t,12,t,g,00093,0,+ +a,17.67,0.0,y,p,j,ff,0.0,f,f,0,f,g,00086,0,- +b,34.58,0.0,?,?,?,?,0.0,f,f,0,f,p,?,0,- +b,18.08,5.5,u,g,k,v,0.5,t,f,0,f,g,00080,0,+ +a,16.92,0.5,u,g,i,v,0.165,f,t,6,t,g,00240,35,- +b,51.33,10.0,u,g,i,bb,0.0,t,t,11,f,g,00000,1249,+ +b,25.00,12.0,u,g,k,v,2.25,t,t,2,t,g,00120,5,- +b,20.42,1.085,u,g,q,v,1.5,f,f,0,f,g,00108,7,- +b,23.75,0.415,y,p,c,v,0.04,f,t,2,f,g,00128,6,- +a,30.00,5.29,u,g,e,dd,2.25,t,t,5,t,g,00099,500,+ +b,44.83,7.0,y,p,c,v,1.625,f,f,0,f,g,00160,2,- +a,25.42,1.125,u,g,q,v,1.29,t,t,2,f,g,00200,0,- +a,19.58,0.665,y,p,c,v,1.0,f,t,1,f,g,02000,2,- +b,31.08,1.5,y,p,w,v,0.04,f,f,0,f,s,00160,0,- +a,20.75,10.335,u,g,cc,h,0.335,t,t,1,t,g,00080,50,+ +b,34.92,2.5,u,g,w,v,0.0,t,f,0,t,g,00239,200,+ +a,58.33,10.0,u,g,q,v,4.0,t,t,14,f,g,00000,1602,+ +a,22.67,0.335,u,g,q,v,0.75,f,f,0,f,s,00160,0,- +a,35.17,3.75,u,g,ff,ff,0.0,f,t,6,f,g,00000,200,- +b,28.67,9.335,u,g,q,h,5.665,t,t,6,f,g,00381,168,+ +b,39.50,1.625,u,g,c,v,1.5,f,f,0,f,g,00000,316,- +?,26.50,2.71,y,p,?,?,0.085,f,f,0,f,s,00080,0,- +b,32.25,0.165,y,p,c,h,3.25,t,t,1,t,g,00432,8000,+ +a,24.58,0.67,u,g,aa,h,1.75,t,f,0,f,g,00400,0,- +b,54.42,0.5,y,p,k,h,3.96,t,f,0,f,g,00180,314,+ +b,18.42,10.415,y,p,aa,v,0.125,t,f,0,f,g,00120,375,- +b,27.25,1.585,u,g,cc,h,1.835,t,t,12,t,g,00583,713,+ +a,26.92,13.5,u,g,q,h,5.0,t,t,2,f,g,00000,5000,+ +b,23.17,11.125,u,g,x,h,0.46,t,t,1,f,g,00100,0,+ +a,15.92,2.875,u,g,q,v,0.085,f,f,0,f,g,00120,0,- +b,16.08,0.75,u,g,c,v,1.75,t,t,5,t,g,00352,690,+ +b,31.58,0.75,y,p,aa,v,3.5,f,f,0,t,g,00320,0,- +a,23.58,11.5,y,p,k,h,3.0,f,f,0,t,g,00020,16,- +b,43.25,3.0,u,g,q,h,6.0,t,t,11,f,g,00080,0,+ +a,24.50,1.04,y,p,ff,ff,0.5,t,t,3,f,g,00180,147,- +b,23.50,3.165,y,p,k,v,0.415,f,t,1,t,g,00280,80,- +b,19.17,9.5,u,g,w,v,1.5,t,f,0,f,g,00120,2206,+ +b,74.83,19.0,y,p,ff,ff,0.04,f,t,2,f,g,00000,351,- +a,48.17,1.335,u,g,i,o,0.335,f,f,0,f,g,00000,120,- +b,25.17,6.0,u,g,c,v,1.0,t,t,3,f,g,00000,0,+ +b,32.92,2.5,u,g,aa,v,1.75,f,t,2,t,g,00720,0,- +b,22.25,0.46,u,g,k,v,0.125,f,f,0,t,g,00280,55,- +a,46.67,0.46,u,g,cc,h,0.415,t,t,11,t,g,00440,6,+ +b,25.25,1.0,u,g,aa,v,0.5,f,f,0,f,g,00200,0,- +b,23.08,0.0,u,g,k,v,1.0,f,t,11,f,s,00000,0,- +b,18.33,1.21,y,p,e,dd,0.0,f,f,0,f,g,00100,0,- +?,20.08,0.125,u,g,q,v,1.0,f,t,1,f,g,00240,768,+ +a,36.75,4.71,u,g,ff,ff,0.0,f,f,0,f,g,00160,0,- +b,20.00,1.25,y,p,k,v,0.125,f,f,0,f,g,00140,4,- +b,27.75,0.585,y,p,cc,v,0.25,t,t,2,f,g,00260,500,+ +b,22.75,11.0,u,g,q,v,2.5,t,t,7,t,g,00100,809,+ +b,26.17,0.835,u,g,cc,v,1.165,f,f,0,f,g,00100,0,- +b,22.50,0.125,y,p,k,v,0.125,f,f,0,f,g,00200,70,- +b,18.58,10.29,u,g,ff,ff,0.415,f,f,0,f,g,00080,0,- +b,27.58,2.04,y,p,aa,v,2.0,t,t,3,t,g,00370,560,+ +b,22.17,0.585,y,p,ff,ff,0.0,f,f,0,f,g,00100,0,- +b,28.17,0.125,y,p,k,v,0.085,f,f,0,f,g,00216,2100,- +b,23.25,1.5,u,g,q,v,2.375,t,t,3,t,g,00000,582,+ +b,?,3.0,y,p,i,bb,7.0,f,f,0,f,g,00000,1,- +b,28.58,3.625,u,g,aa,v,0.25,f,f,0,t,g,00100,0,- +a,60.92,5.0,u,g,aa,v,4.0,t,t,4,f,g,00000,99,+ +a,26.58,2.54,y,p,ff,ff,0.0,f,f,0,t,g,00180,60,- +b,30.58,2.71,y,p,m,v,0.125,f,f,0,t,s,00080,0,- +b,22.00,0.79,u,g,w,v,0.29,f,t,1,f,g,00420,283,- +a,40.33,7.54,y,p,q,h,8.0,t,t,14,f,g,00000,2300,+ +b,51.42,0.04,u,g,x,h,0.04,t,f,0,f,g,00000,3000,+ +b,63.33,0.54,u,g,c,v,0.585,t,t,3,t,g,00180,0,- +a,36.58,0.29,u,g,ff,ff,0.0,f,t,10,f,g,00200,18,- +a,29.50,1.085,y,p,x,v,1.0,f,f,0,f,g,00280,13,- +b,40.00,6.5,u,g,aa,bb,3.5,t,t,1,f,g,00000,500,+ +a,44.17,6.665,u,g,q,v,7.375,t,t,3,t,g,00000,0,+ +b,32.67,9.0,y,p,w,h,5.25,t,f,0,t,g,00154,0,+ +b,23.00,0.625,y,p,aa,v,0.125,t,f,0,f,g,00180,1,- +b,22.33,11.0,u,g,w,v,2.0,t,t,1,f,g,00080,278,+ +b,38.17,10.125,u,g,x,v,2.5,t,t,6,f,g,00520,196,+ +b,41.17,1.25,y,p,w,v,0.25,f,f,0,f,g,00000,195,- +b,16.33,4.085,u,g,i,h,0.415,f,f,0,t,g,00120,0,- +b,48.17,7.625,u,g,w,h,15.5,t,t,12,f,g,00000,790,+ +b,?,0.5,u,g,c,bb,0.835,t,f,0,t,s,00320,0,- +b,26.33,13.0,u,g,e,dd,0.0,f,f,0,t,g,00140,1110,- +b,33.67,1.25,u,g,w,v,1.165,f,f,0,f,g,00120,0,- +b,16.25,0.835,u,g,m,v,0.085,t,f,0,f,s,00200,0,- +b,35.75,2.415,u,g,w,v,0.125,f,t,2,f,g,00220,1,- +b,?,0.625,u,g,k,v,0.25,f,f,0,f,g,00380,2010,- +b,20.75,5.085,y,p,j,v,0.29,f,f,0,f,g,00140,184,- +a,24.75,12.5,u,g,aa,v,1.5,t,t,12,t,g,00120,567,+ +a,57.08,19.5,u,g,c,v,5.5,t,t,7,f,g,00000,3000,+ +b,24.83,2.75,u,g,c,v,2.25,t,t,6,f,g,?,600,+ +a,27.17,1.25,u,g,ff,ff,0.0,f,t,1,f,g,00092,300,- +b,48.58,0.205,y,p,k,v,0.25,t,t,11,f,g,00380,2732,+ +b,45.00,8.5,u,g,cc,h,14.0,t,t,1,t,g,00088,2000,+ +a,22.25,1.25,y,p,ff,ff,3.25,f,f,0,f,g,00280,0,- +b,47.83,4.165,u,g,x,bb,0.085,f,f,0,t,g,00520,0,- +a,28.42,3.5,u,g,w,v,0.835,t,f,0,f,s,00280,0,+ +a,51.92,6.5,u,g,i,bb,3.085,f,f,0,t,g,00073,0,- +b,36.33,3.79,u,g,w,v,1.165,t,f,0,t,g,00200,0,- +a,33.92,1.585,y,p,ff,ff,0.0,t,f,0,f,g,00320,0,- +a,18.92,9.25,y,p,c,v,1.0,t,t,4,t,g,00080,500,+ +b,23.75,12.0,u,g,c,v,2.085,f,f,0,f,s,00080,0,- +a,22.50,8.5,u,g,q,v,1.75,t,t,10,f,g,00080,990,- +b,29.25,13.0,u,g,d,h,0.5,f,f,0,f,g,00228,0,- +a,25.33,2.085,u,g,c,h,2.75,t,f,0,t,g,00360,1,- +b,22.58,1.5,y,p,aa,v,0.54,f,f,0,t,g,00120,67,- +a,17.92,0.54,u,g,c,v,1.75,f,t,1,t,g,00080,5,- +b,21.00,4.79,y,p,w,v,2.25,t,t,1,t,g,00080,300,+ +a,32.00,6.0,u,g,d,v,1.25,f,f,0,f,g,00272,0,- +b,39.58,5.0,u,g,ff,ff,0.0,f,t,2,f,g,00017,1,- +a,25.00,0.875,u,g,x,h,1.04,t,f,0,t,g,00160,5860,+ +b,24.58,12.5,u,g,w,v,0.875,t,f,0,t,g,00260,0,- +b,33.50,1.75,u,g,x,h,4.5,t,t,4,t,g,00253,857,+ +a,21.50,6.0,u,g,aa,v,2.5,t,t,3,f,g,00080,918,+ +b,26.25,1.54,u,g,w,v,0.125,f,f,0,f,g,00100,0,- +b,27.67,13.75,u,g,w,v,5.75,t,f,0,t,g,00487,500,+ +b,30.08,1.04,y,p,i,bb,0.5,t,t,10,t,g,00132,28,- +b,17.50,22.0,l,gg,ff,o,0.0,f,f,0,t,p,00450,100000,+ +b,31.25,2.835,u,g,ff,ff,0.0,f,t,5,f,g,00176,146,- +b,36.42,0.75,y,p,d,v,0.585,f,f,0,f,g,00240,3,- +b,32.42,3.0,u,g,d,v,0.165,f,f,0,t,g,00120,0,- +a,20.75,9.54,u,g,i,v,0.04,f,f,0,f,g,00200,1000,- +b,17.67,4.46,u,g,c,v,0.25,f,f,0,f,s,00080,0,- +b,28.67,14.5,u,g,d,v,0.125,f,f,0,f,g,00000,286,- +b,22.08,11.46,u,g,k,v,1.585,f,f,0,t,g,00100,1212,- +a,37.75,5.5,u,g,q,v,0.125,t,f,0,t,g,00228,0,+ +a,23.58,0.83,u,g,q,v,0.415,f,t,1,t,g,00200,11,- +a,28.58,3.75,u,g,c,v,0.25,f,t,1,t,g,00040,154,- +b,17.42,6.5,u,g,i,v,0.125,f,f,0,f,g,00060,100,- +b,26.75,4.5,y,p,c,bb,2.5,f,f,0,f,g,00200,1210,- +b,31.33,19.5,u,g,c,v,7.0,t,t,16,f,g,00000,5000,+ +a,20.08,1.25,u,g,c,v,0.0,f,f,0,f,g,00000,0,- +b,40.92,0.5,y,p,m,v,0.5,f,f,0,t,g,00130,0,-