Skip to content

add automl experiment api && cfo tuner #6140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
michaelgsharp merged 28 commits into from
Apr 21, 2022
Merged

add automl experiment api && cfo tuner #6140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
c15a009
add sweepable estimator generator
LittleLittleCloud Mar 6, 2022
44da68a
add runtime
LittleLittleCloud Mar 6, 2022
365dd6a
enable sweepabe estimator factory generator
LittleLittleCloud Mar 6, 2022
50d2c28
add tests for sweepable estimator pipeline and multi-model pipeline
LittleLittleCloud Mar 14, 2022
8682502
add sweepable extension tests and more tests
LittleLittleCloud Mar 14, 2022
e5ac469
add automl experiment
LittleLittleCloud Mar 15, 2022
4f5567e
fix bugs
LittleLittleCloud Mar 15, 2022
c057946
add append
LittleLittleCloud Mar 16, 2022
3336402
add titanic example
LittleLittleCloud Mar 16, 2022
0fe559e
use di to inject services
LittleLittleCloud Mar 23, 2022
9a48411
add cfo, random and grid search tuners and tests
LittleLittleCloud Mar 23, 2022
80c574a
Merge branch 'main' into u/xiaoyun/addAutoMLExperiment
LittleLittleCloud Mar 28, 2022
735264f
Update Microsoft.ML.AutoML.Samples.csproj
LittleLittleCloud Mar 28, 2022
311b810
Update Program.cs
LittleLittleCloud Mar 28, 2022
002c083
Update Assembly.cs
LittleLittleCloud Mar 28, 2022
6301ae2
rm unnecessary change
LittleLittleCloud Mar 28, 2022
f7c0a68
add regression and multi-classification runner
LittleLittleCloud Mar 29, 2022
fbccf2a
fix bug
LittleLittleCloud Mar 29, 2022
7de0768
update IMetricSettings
LittleLittleCloud Apr 4, 2022
8fc794b
checkout assembly
LittleLittleCloud Apr 4, 2022
b3dfbf7
fix comments
LittleLittleCloud Apr 5, 2022
17e7b93
fix comments and build
LittleLittleCloud Apr 7, 2022
7ac939e
Merge branch 'main' into u/xiaoyun/addAutoMLExperiment
LittleLittleCloud Apr 13, 2022
b082275
fix bug
LittleLittleCloud Apr 14, 2022
f2320ff
update
LittleLittleCloud Apr 19, 2022
1c85cfa
add comment to estimator cost
LittleLittleCloud Apr 19, 2022
eaef130
Merge branch 'u/xiaoyun/addAutoMLExperiment' of https://github.com/Li…
LittleLittleCloud Apr 19, 2022
c1fc20b
rename cfoTunerFactory to CostFrugalFactory
LittleLittleCloud Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions eng/Versions.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
<MoqVersion>4.8.3</MoqVersion>
<MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
<McMasterExtensionsCommandLineUtils>2.3.0</McMasterExtensionsCommandLineUtils>
<MicrosoftExtensionsDependencyInjectionVersion>6.0.0</MicrosoftExtensionsDependencyInjectionVersion>
<NewtonsoftJsonVersion>13.0.1</NewtonsoftJsonVersion>
<SystemTextJsonVersion>6.0.1</SystemTextJsonVersion>
<NuGetVersioningVersion>4.4.0</NuGetVersioningVersion>
Expand Down
40 changes: 22 additions & 18 deletions src/Microsoft.ML.AutoML/API/AutoCatalog.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,11 @@ internal SweepableEstimator CreateSweepableEstimator<T>(Func<MLContext, T, IEsti
return new SweepableEstimator((MLContext context, Parameter param) => factory(context, param.AsType<T>()), ss);
}

internal AutoMLExperiment CreateExperiment()
{
return new AutoMLExperiment(this._context, new AutoMLExperiment.AutoMLExperimentSettings());
}

internal SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true,
FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null,
SearchSpace<FastTreeOption> fastTreeSearchSpace = null, SearchSpace<LgbmOption> lgbmSearchSpace = null, SearchSpace<FastForestOption> fastForestSearchSpace = null, SearchSpace<LbfgsOption> lbfgsSearchSpace = null, SearchSpace<SdcaOption> sdcaSearchSpace = null)
Expand All @@ -306,7 +311,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa
fastTreeOption.LabelColumnName = labelColumnName;
fastTreeOption.FeatureColumnName = featureColumnName;
fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>()));
res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>(fastTreeOption)));
}

if (useFastForest)
Expand All @@ -315,7 +320,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa
fastForestOption.LabelColumnName = labelColumnName;
fastForestOption.FeatureColumnName = featureColumnName;
fastForestOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace<FastForestOption>()));
res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace<FastForestOption>(fastForestOption)));
}

if (useLgbm)
Expand All @@ -324,7 +329,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa
lgbmOption.LabelColumnName = labelColumnName;
lgbmOption.FeatureColumnName = featureColumnName;
lgbmOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace<LgbmOption>()));
res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace<LgbmOption>(lgbmOption)));
}

if (useLbfgs)
Expand All @@ -333,7 +338,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa
lbfgsOption.LabelColumnName = labelColumnName;
lbfgsOption.FeatureColumnName = featureColumnName;
lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>()));
res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>(lbfgsOption)));
}

if (useSdca)
Expand All @@ -342,7 +347,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa
sdcaOption.LabelColumnName = labelColumnName;
sdcaOption.FeatureColumnName = featureColumnName;
sdcaOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>()));
res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>(sdcaOption)));
}

return res.ToArray();
Expand All @@ -360,7 +365,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau
fastTreeOption.LabelColumnName = labelColumnName;
fastTreeOption.FeatureColumnName = featureColumnName;
fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>()));
res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>(fastTreeOption)));
}

if (useFastForest)
Expand All @@ -369,7 +374,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau
fastForestOption.LabelColumnName = labelColumnName;
fastForestOption.FeatureColumnName = featureColumnName;
fastForestOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace<FastForestOption>()));
res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace<FastForestOption>(fastForestOption)));
}

if (useLgbm)
Expand All @@ -378,7 +383,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau
lgbmOption.LabelColumnName = labelColumnName;
lgbmOption.FeatureColumnName = featureColumnName;
lgbmOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace<LgbmOption>()));
res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace<LgbmOption>(lgbmOption)));
}

if (useLbfgs)
Expand All @@ -387,8 +392,8 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau
lbfgsOption.LabelColumnName = labelColumnName;
lbfgsOption.FeatureColumnName = featureColumnName;
lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>()));
res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>()));
res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>(lbfgsOption)));
res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>(lbfgsOption)));
}

if (useSdca)
Expand All @@ -397,8 +402,8 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau
sdcaOption.LabelColumnName = labelColumnName;
sdcaOption.FeatureColumnName = featureColumnName;
sdcaOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>()));
res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>()));
res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>(sdcaOption)));
res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>(sdcaOption)));
}

return res.ToArray();
Expand All @@ -416,8 +421,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN
fastTreeOption.LabelColumnName = labelColumnName;
fastTreeOption.FeatureColumnName = featureColumnName;
fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>()));
res.Add(SweepableEstimatorFactory.CreateFastTreeTweedieRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>()));
res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace<FastTreeOption>(fastTreeOption)));
}

if (useFastForest)
Expand All @@ -426,7 +430,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN
fastForestOption.LabelColumnName = labelColumnName;
fastForestOption.FeatureColumnName = featureColumnName;
fastForestOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace<FastForestOption>()));
res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace<FastForestOption>(fastForestOption)));
}

if (useLgbm)
Expand All @@ -435,7 +439,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN
lgbmOption.LabelColumnName = labelColumnName;
lgbmOption.FeatureColumnName = featureColumnName;
lgbmOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace<LgbmOption>()));
res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace<LgbmOption>(lgbmOption)));
}

if (useLbfgs)
Expand All @@ -444,7 +448,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN
lbfgsOption.LabelColumnName = labelColumnName;
lbfgsOption.FeatureColumnName = featureColumnName;
lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>()));
res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace<LbfgsOption>(lbfgsOption)));
}

if (useSdca)
Expand All @@ -453,7 +457,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN
sdcaOption.LabelColumnName = labelColumnName;
sdcaOption.FeatureColumnName = featureColumnName;
sdcaOption.ExampleWeightColumnName = exampleWeightColumnName;
res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>()));
res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace<SdcaOption>(sdcaOption)));
}

return res.ToArray();
Expand Down
8 changes: 8 additions & 0 deletions src/Microsoft.ML.AutoML/API/SweepableExtension.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ public static MultiModelPipeline Append(this IEstimator<ITransformer> estimator,
return multiModelPipeline;
}

public static MultiModelPipeline Append(this MultiModelPipeline pipeline, IEstimator<ITransformer> estimator)
{
var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace());
var multiModelPipeline = pipeline.Append(sweepableEstimator);

return multiModelPipeline;
}

public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators)
{
var multiModelPipeline = new MultiModelPipeline();
Expand Down
Loading