From c15a009f188ed46ec3e26780760b67e7e0d3f290 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 6 Mar 2022 11:37:48 -0800 Subject: [PATCH 01/25] add sweepable estimator generator --- .../CodeGen/code_gen_flag.json | 2 +- ...Microsoft.ML.AutoML.SourceGenerator.csproj | 18 + .../SweepableEstimatorGenerator.cs | 88 ++++ .../Template/SweepableEstimator.cs | 387 ++++++++++++++++++ .../Template/SweepableEstimator.tt | 81 ++++ .../Template/SweepableEstimator_T_.cs | 358 ++++++++++++++++ .../Template/SweepableEstimator_T_.tt | 60 +++ .../Utils.cs | 55 +++ 8 files changed, 1048 insertions(+), 1 deletion(-) create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt diff --git a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json index f85ae7c22d..cddb6933a5 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json +++ b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json @@ -3,5 +3,5 @@ "CodeGenCatalogGenerator": false, "EstimatorTypeGenerator": true, "SearchSpaceGenerator": true, - "SweepableEstimatorGenerator": false + "SweepableEstimatorGenerator": true } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj index 271f5d8214..df17513e14 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj @@ -41,6 +41,16 @@ True SearchSpace.tt + + True + True + SweepableEstimator.tt + + + True + True + SweepableEstimator_T_.tt + @@ -52,6 +62,14 @@ TextTemplatingFilePreprocessor SearchSpace.cs + + TextTemplatingFilePreprocessor + SweepableEstimator.cs + + + TextTemplatingFilePreprocessor + SweepableEstimator_T_.cs + diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs new file mode 100644 index 0000000000..15ab0cf4d2 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; +using Microsoft.ML.AutoML.SourceGenerator; +using SweepableEstimator = Microsoft.ML.AutoML.SourceGenerator.Template.SweepableEstimator; +using SweepableEstimatorT = Microsoft.ML.AutoML.SourceGenerator.Template.SweepableEstimator_T_; +namespace Microsoft.ML.ModelBuilder.SweepableEstimator.CodeGenerator +{ + [Generator] + public class SweepableEstimatorGenerator : ISourceGenerator + { + private const string SweepableEstimatorAttributeDisplayName = Constant.CodeGeneratorNameSpace + "." + "SweepableEstimatorAttribute"; + + public void Execute(GeneratorExecutionContext context) + { + if (context.AdditionalFiles.Where(f => f.Path.Contains("code_gen_flag.json")).First() is AdditionalText text) + { + var json = text.GetText().ToString(); + var flags = JsonSerializer.Deserialize>(json); + if (flags.TryGetValue(nameof(SweepableEstimatorGenerator), out var res) && res == false) + { + return; + } + } + + var estimators = context.AdditionalFiles.Where(f => f.Path.Contains("trainer-estimators.json") || f.Path.Contains("transformer-estimators.json")) + .SelectMany(file => Utils.GetEstimatorsFromJson(file.GetText().ToString()).Estimators) + .ToArray(); + + var code = estimators.SelectMany(e => e.EstimatorTypes.Select(eType => (e, eType, Utils.CreateEstimatorName(e.FunctionName, eType))) + .Select(x => + { + if (x.e.SearchOption == null) + { + return + (x.Item3, + new AutoML.SourceGenerator.Template.SweepableEstimator() + { + NameSpace = Constant.CodeGeneratorNameSpace, + UsingStatements = x.e.UsingStatements, + ArgumentsList = x.e.ArgumentsList, + ClassName = x.Item3, + FunctionName = x.e.FunctionName, + NugetDependencies = x.e.NugetDependencies, + Type = x.eType, + }.TransformText()); + } + else + { + return + (x.Item3, + new SweepableEstimatorT() + { + NameSpace = Constant.CodeGeneratorNameSpace, + UsingStatements = x.e.UsingStatements, + ArgumentsList = x.e.ArgumentsList, + ClassName = x.Item3, + FunctionName = x.e.FunctionName, + NugetDependencies = x.e.NugetDependencies, + Type = x.eType, + TOption = Utils.ToTitleCase(x.e.SearchOption), + }.TransformText()); + } + })); + + foreach (var c in code) + { + context.AddSource(c.Item1 + ".cs", SourceText.From(c.Item2, Encoding.UTF8)); + } + } + + public void Initialize(GeneratorInitializationContext context) + { + return; + //context.RegisterForPostInitialization(i => i.AddSource(nameof(SweepableEstimatorAttribute), SweepableEstimatorAttribute)); + } + } +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs new file mode 100644 index 0000000000..ceb30b1082 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs @@ -0,0 +1,387 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version: 17.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +namespace Microsoft.ML.AutoML.SourceGenerator.Template +{ + using System.Linq; + using System.Text; + using System.Collections.Generic; + using System; + + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class SweepableEstimator : SweepableEstimatorBase + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write(@" +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; + +namespace "); + this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); + this.Write("\r\n{\r\n internal partial class "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(" : SweepableEstimator\r\n {\r\n public "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write("()\r\n {\r\n this.EstimatorType = EstimatorType."); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(";\r\n }\r\n \r\n"); + foreach(var arg in ArgumentsList){ + var typeAttributeName = Utils.CapitalFirstLetter(arg.ArgumentType); + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); + this.Write(" ["); + this.Write(this.ToStringHelper.ToStringWithCulture(typeAttributeName)); + this.Write("]\r\n [JsonProperty(NullValueHandling=NullValueHandling.Ignore)]\r\n pu" + + "blic string "); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(" { get; set; }\r\n\r\n"); +} + this.Write(" public override IEnumerable CSharpUsingStatements \r\n {\r\n " + + " get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); + this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + + " {\r\n get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); + this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + + " get => \""); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); + this.Write("."); + this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); + this.Write("\";\r\n }\r\n\r\n public override string ToCSharpCode()\r\n {\r\n"); +if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){ + this.Write(" return this.BuildCSharpCodeForTrainers();\r\n"); +}else if (Type == "OneVersusAll"){ + this.Write(" return this.BuildCSharpCodeForOva();\r\n"); +}else{ + this.Write(" return this.BuildCSharpCodeForTransformers();\r\n"); +} + this.Write(" }\r\n\r\n internal override void UpdatePropertiesFromOptions(Dictionar" + + "y options)\r\n {\r\n"); + foreach(var arg in ArgumentsList){ + var typeAttributeName = arg.ArgumentType; + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); + if(typeAttributeName == "integer"){ + this.Write(" this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(" = options.ContainsKey(nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write("))? this.ToIntegerString(options[nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(")]) : this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(";\r\n "); +}else{ + this.Write(" this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(" = options.ContainsKey(nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write("))? options[nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(")] : this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(";\r\n "); +}} + this.Write(" }\r\n }\r\n}\r\n\r\n"); + return this.GenerationEnvironment.ToString(); + } + +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} + + } + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class SweepableEstimatorBase + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + protected System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; + } + set + { + this.generationEnvironmentField = value; + } + } + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + } + return this.errorsField; + } + } + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); + } + return this.indentLengthsField; + } + } + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; + } + } + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; + } + set + { + this.sessionField = value; + } + } + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } + } + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; + } + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); + } + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); + } + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); + } + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; + } + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; + } + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; + } + set + { + if ((value != null)) + { + this.formatProviderField = value; + } + } + } + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { + throw new global::System.ArgumentNullException("objectToConvert"); + } + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); + } + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); + } + } + } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion + } + #endregion +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt new file mode 100644 index 0000000000..b4e9a68558 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt @@ -0,0 +1,81 @@ +<#@ template language="C#" linePragmas="false" visibility = "internal"#> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> + +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; + +namespace <#=NameSpace#> +{ + internal partial class <#=ClassName#> : SweepableEstimator + { + public <#=ClassName#>() + { + this.EstimatorType = EstimatorType.<#=ClassName#>; + } + +<# foreach(var arg in ArgumentsList){ + var typeAttributeName = Utils.CapitalFirstLetter(arg.ArgumentType); + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName);#> + [<#=typeAttributeName#>] + [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] + public string <#=propertyName#> { get; set; } + +<#}#> + public override IEnumerable CSharpUsingStatements + { + get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; + } + + public override IEnumerable NugetDependencies + { + get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; + } + + public override string FunctionName + { + get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; + } + + public override string ToCSharpCode() + { +<#if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){#> + return this.BuildCSharpCodeForTrainers(); +<#}else if (Type == "OneVersusAll"){#> + return this.BuildCSharpCodeForOva(); +<#}else{#> + return this.BuildCSharpCodeForTransformers(); +<#}#> + } + + internal override void UpdatePropertiesFromOptions(Dictionary options) + { +<# foreach(var arg in ArgumentsList){ + var typeAttributeName = arg.ArgumentType; + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); + if(typeAttributeName == "integer"){#> + this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? this.ToIntegerString(options[nameof(this.<#=propertyName#>)]) : this.<#=propertyName#>; + <#}else{#> + this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? options[nameof(this.<#=propertyName#>)] : this.<#=propertyName#>; + <#}}#> + } + } +} + +<#+ +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} +#> \ No newline at end of file diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs new file mode 100644 index 0000000000..9b1327aacb --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs @@ -0,0 +1,358 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version: 17.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +namespace Microsoft.ML.AutoML.SourceGenerator.Template +{ + using System.Linq; + using System.Text; + using System.Collections.Generic; + using System; + + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class SweepableEstimator_T_ : SweepableEstimator_T_Base + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write(@" +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; +using Microsoft.ML.SearchSpace; + +namespace "); + this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); + this.Write("\r\n{\r\n internal partial class "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(" : SweepableEstimator<"); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write(">\r\n {\r\n public "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write("("); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write(" defaultOption, SearchSpace<"); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write("> searchSpace = null)\r\n {\r\n this.Parameter = defaultOption;\r\n " + + " this.SearchSpace = searchSpace;\r\n this.EstimatorType = Esti" + + "matorType."); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(";\r\n }\r\n\r\n internal "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write("()\r\n {\r\n this.EstimatorType = EstimatorType."); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(";\r\n this.Parameter = new "); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write("();\r\n }\r\n \r\n public override IEnumerable CSharpUsingStat" + + "ements \r\n {\r\n get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); + this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + + " {\r\n get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); + this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + + " get => \""); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); + this.Write("."); + this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); + this.Write("\";\r\n }\r\n }\r\n}\r\n\r\n"); + return this.GenerationEnvironment.ToString(); + } + +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} +public string TOption {get; set;} + + } + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class SweepableEstimator_T_Base + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + protected System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; + } + set + { + this.generationEnvironmentField = value; + } + } + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + } + return this.errorsField; + } + } + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); + } + return this.indentLengthsField; + } + } + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; + } + } + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; + } + set + { + this.sessionField = value; + } + } + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } + } + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; + } + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); + } + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); + } + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); + } + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; + } + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; + } + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; + } + set + { + if ((value != null)) + { + this.formatProviderField = value; + } + } + } + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { + throw new global::System.ArgumentNullException("objectToConvert"); + } + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); + } + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); + } + } + } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion + } + #endregion +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt new file mode 100644 index 0000000000..ff6b3e5974 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt @@ -0,0 +1,60 @@ +<#@ template language="C#" linePragmas="false" visibility = "internal"#> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> + +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; +using Microsoft.ML.SearchSpace; + +namespace <#=NameSpace#> +{ + internal partial class <#=ClassName#> : SweepableEstimator<<#=TOption#>> + { + public <#=ClassName#>(<#=TOption#> defaultOption, SearchSpace<<#=TOption#>> searchSpace = null) + { + this.Parameter = defaultOption; + this.SearchSpace = searchSpace; + this.EstimatorType = EstimatorType.<#=ClassName#>; + } + + internal <#=ClassName#>() + { + this.EstimatorType = EstimatorType.<#=ClassName#>; + this.Parameter = new <#=TOption#>(); + } + + public override IEnumerable CSharpUsingStatements + { + get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; + } + + public override IEnumerable NugetDependencies + { + get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; + } + + public override string FunctionName + { + get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; + } + } +} + +<#+ +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} +public string TOption {get; set;} +#> \ No newline at end of file diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs index c71e7a0f90..e6897a8d6a 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -70,5 +71,59 @@ public static string ToTitleCase(string str) { return string.Join(string.Empty, str.Split('_', ' ', '-').Select(x => CapitalFirstLetter(x))); } + + public static string GetPrefix(string estimatorType) + { + if (estimatorType == "BinaryClassification") + { + return "BinaryClassification.Trainers"; + } + if (estimatorType == "MultiClassification") + { + return "MulticlassClassification.Trainers"; + } + if (estimatorType == "Regression") + { + return "Regression.Trainers"; + } + if (estimatorType == "Ranking") + { + return "Ranking.Trainers"; + } + if (estimatorType == "OneVersusAll") + { + return "BinaryClassification.Trainers"; + } + if (estimatorType == "Recommendation") + { + return "Recommendation().Trainers"; + } + if (estimatorType == "Transforms") + { + return "Transforms"; + } + if (estimatorType == "Categorical") + { + return "Transforms.Categorical"; + } + if (estimatorType == "Conversion") + { + return "Transforms.Conversion"; + } + if (estimatorType == "Text") + { + return "Transforms.Text"; + } + if (estimatorType == "Calibrators") + { + return "BinaryClassification.Calibrators"; + } + if (estimatorType == "Forecasting") + { + return "Forecasting"; + } + + throw new NotImplementedException(); + } } } From 44da68a1a9d143123924322ed7a0987f5c7a92f8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 6 Mar 2022 12:01:44 -0800 Subject: [PATCH 02/25] add runtime --- src/Microsoft.ML.AutoML/AutoMlUtils.cs | 34 +++++++ .../Microsoft.ML.AutoML.csproj | 10 +- .../Estimators/ApplyOnnx.cs | 18 ++++ .../Estimators/Calibrators.cs | 18 ++++ .../Estimators/Concatenate.cs | 14 +++ .../Estimators/FastForest.cs | 63 ++++++++++++ .../SweepableEstimator/Estimators/FastTree.cs | 96 +++++++++++++++++++ .../Estimators/FeaturizeText.cs | 14 +++ .../Estimators/ForecastBySsa.cs | 21 ++++ .../SweepableEstimator/Estimators/Images.cs | 47 +++++++++ .../SweepableEstimator/Estimators/Lbfgs.cs | 81 ++++++++++++++++ .../SweepableEstimator/Estimators/LightGbm.cs | 92 ++++++++++++++++++ .../Estimators/MapValueToKey.cs | 22 +++++ .../Estimators/MatrixFactorization.cs | 14 +++ .../Estimators/NormalizeMinMax.cs | 16 ++++ .../Estimators/OneHotEncoding.cs | 24 +++++ .../Estimators/ReplaceMissingValue.cs | 15 +++ .../SweepableEstimator/Estimators/Sdca.cs | 81 ++++++++++++++++ .../Estimators/TypeConvert.cs | 15 +++ .../SweepableEstimator/SweepableEstimator.cs | 12 --- .../Template/SweepableEstimator.cs | 48 ++-------- .../Template/SweepableEstimator.tt | 29 +----- .../Template/SweepableEstimator_T_.cs | 20 ++-- .../Template/SweepableEstimator_T_.tt | 10 +- 24 files changed, 716 insertions(+), 98 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs diff --git a/src/Microsoft.ML.AutoML/AutoMlUtils.cs b/src/Microsoft.ML.AutoML/AutoMlUtils.cs index 8e9ad11a22..07ee28c6c3 100644 --- a/src/Microsoft.ML.AutoML/AutoMlUtils.cs +++ b/src/Microsoft.ML.AutoML/AutoMlUtils.cs @@ -3,12 +3,46 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Threading; namespace Microsoft.ML.AutoML { internal static class AutoMlUtils { + private const string MLNetMaxThread = "MLNET_MAX_THREAD"; + public static readonly ThreadLocal Random = new ThreadLocal(() => new Random()); + + /// + /// Return number of thread if MLNET_MAX_THREAD is set, otherwise return null. + /// + public static int? GetNumberOfThreadFromEnvrionment() + { + var res = Environment.GetEnvironmentVariable(MLNetMaxThread); + + if (int.TryParse(res, out var numberOfThread)) + { + return numberOfThread; + } + + return null; + } + + public static InputOutputColumnPair[] CreateInputOutputColumnPairsFromStrings(string[] inputs, string[] outputs) + { + if (inputs.Length != outputs.Length) + { + throw new Exception("inputs and outputs count must match"); + } + + var res = new List(); + for (int i = 0; i != inputs.Length; ++i) + { + res.Add(new InputOutputColumnPair(outputs[i], inputs[i])); + } + + return res.ToArray(); + } } } diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 46632e232a..6d79838dff 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -14,11 +14,13 @@ all + all true + @@ -43,13 +45,13 @@ - + - + - - + + diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs new file mode 100644 index 0000000000..daabd21e6b --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ApplyOnnxModel + { + public override IEstimator BuildFromOption(MLContext context, ApplyOnnxModelOption param) + { + return context.Transforms.ApplyOnnxModel(outputColumnName: param.OutputColumnName, inputColumnName: param.InputColumnName, modelFile: param.ModelFile, gpuDeviceId: param.GpuDeviceId, fallbackToCpu: param.FallbackToCpu); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs new file mode 100644 index 0000000000..f59c7264de --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class Naive + { + public override IEstimator BuildFromOption(MLContext context, NaiveOption param) + { + return context.BinaryClassification.Calibrators.Naive(param.LabelColumnName, param.ScoreColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs new file mode 100644 index 0000000000..aa085543af --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class Concatenate + { + public override IEstimator BuildFromOption(MLContext context, ConcatOption param) + { + return context.Transforms.Concatenate(param.OutputColumnName, param.InputColumnNames); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs new file mode 100644 index 0000000000..c9e900f023 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers.FastTree; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class FastForestOva + { + public override IEstimator BuildFromOption(MLContext context, FastForestOption param) + { + var option = new FastForestBinaryTrainer.Options() + { + NumberOfTrees = param.NumberOfTrees, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + FeatureFraction = param.FeatureFraction, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastForest(option), labelColumnName: param.LabelColumnName); + } + } + + internal partial class FastForestRegression + { + public override IEstimator BuildFromOption(MLContext context, FastForestOption param) + { + var option = new FastForestRegressionTrainer.Options() + { + NumberOfTrees = param.NumberOfTrees, + FeatureFraction = param.FeatureFraction, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.Regression.Trainers.FastForest(option); + } + } + + internal partial class FastForestBinary + { + public override IEstimator BuildFromOption(MLContext context, FastForestOption param) + { + var option = new FastForestBinaryTrainer.Options() + { + NumberOfTrees = param.NumberOfTrees, + NumberOfLeaves = param.NumberOfLeaves, + FeatureFraction = param.FeatureFraction, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.BinaryClassification.Trainers.FastForest(option); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs new file mode 100644 index 0000000000..cc6ae89136 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs @@ -0,0 +1,96 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers.FastTree; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class FastTreeOva + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeBinaryTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastTree(option), labelColumnName: param.LabelColumnName); + } + } + + internal partial class FastTreeRegression + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeRegressionTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.Regression.Trainers.FastTree(option); + } + } + + internal partial class FastTreeTweedieRegression + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeTweedieTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.Regression.Trainers.FastTreeTweedie(option); + } + } + + internal partial class FastTreeBinary + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeBinaryTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.BinaryClassification.Trainers.FastTree(option); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs new file mode 100644 index 0000000000..67c4905ebf --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class FeaturizeText + { + public override IEstimator BuildFromOption(MLContext context, FeaturizeTextOption param) + { + return context.Transforms.Text.FeaturizeText(param.OutputColumnName, param.InputColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs new file mode 100644 index 0000000000..993ffdaad4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ForecastBySsa + { + public override IEstimator BuildFromOption(MLContext context, SsaOption param) + { + if (param.SeriesLength <= param.WindowSize || param.TrainSize <= 2 * param.WindowSize) + { + throw new Exception("ForecastBySsa param check error"); + } + + return context.Forecasting.ForecastBySsa(param.OutputColumnName, param.InputColumnName, param.WindowSize, param.SeriesLength, param.TrainSize, param.Horizon, confidenceLowerBoundColumn: param.ConfidenceLowerBoundColumn, confidenceUpperBoundColumn: param.ConfidenceUpperBoundColumn); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs new file mode 100644 index 0000000000..6df898e0f3 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class LoadImages + { + public override IEstimator BuildFromOption(MLContext context, LoadImageOption param) + { + return context.Transforms.LoadImages(param.OutputColumnName, param.ImageFolder, param.InputColumnName); + } + } + + internal partial class LoadRawImageBytes + { + public override IEstimator BuildFromOption(MLContext context, LoadImageOption param) + { + return context.Transforms.LoadRawImageBytes(param.OutputColumnName, param.ImageFolder, param.InputColumnName); + } + } + + internal partial class ResizeImages + { + public override IEstimator BuildFromOption(MLContext context, ResizeImageOption param) + { + return context.Transforms.ResizeImages(param.OutputColumnName, param.ImageWidth, param.ImageHeight, param.InputColumnName, param.Resizing, param.CropAnchor); + } + } + + internal partial class ExtractPixels + { + public override IEstimator BuildFromOption(MLContext context, ExtractPixelsOption param) + { + return context.Transforms.ExtractPixels(param.OutputColumnName, param.InputColumnName, param.ColorsToExtract, param.OrderOfExtraction, outputAsFloatArray: param.OutputAsFloatArray); + } + } + + internal partial class ImageClassificationMulti + { + public override IEstimator BuildFromOption(MLContext context, ImageClassificationOption param) + { + + return context.MulticlassClassification.Trainers.ImageClassification(param.LabelColumnName, param.FeatureColumnName, param.ScoreColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs new file mode 100644 index 0000000000..8480b0e6ee --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class LbfgsMaximumEntropyMulti + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsMaximumEntropyMulticlassTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.MulticlassClassification.Trainers.LbfgsMaximumEntropy(option); + } + } + + internal partial class LbfgsPoissonRegressionRegression + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsPoissonRegressionTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.Regression.Trainers.LbfgsPoissonRegression(option); + } + } + + internal partial class LbfgsLogisticRegressionBinary + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsLogisticRegressionBinaryTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.BinaryClassification.Trainers.LbfgsLogisticRegression(option); + } + } + + internal partial class LbfgsLogisticRegressionOva + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsLogisticRegressionBinaryTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + var binaryTrainer = context.BinaryClassification.Trainers.LbfgsLogisticRegression(option); + return context.MulticlassClassification.Trainers.OneVersusAll(binaryTrainer, param.LabelColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs new file mode 100644 index 0000000000..7164d00907 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers.LightGbm; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class LightGbmMulti + { + public override IEstimator BuildFromOption(MLContext context, LgbmOption param) + { + var option = new LightGbmMulticlassTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfIterations = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + Booster = new GradientBooster.Options() + { + SubsampleFraction = param.SubsampleFraction, + FeatureFraction = param.FeatureFraction, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + }, + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + }; + + return context.MulticlassClassification.Trainers.LightGbm(option); + } + } + + internal partial class LightGbmBinary + { + public override IEstimator BuildFromOption(MLContext context, LgbmOption param) + { + var option = new LightGbmBinaryTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfIterations = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + Booster = new GradientBooster.Options() + { + SubsampleFraction = param.SubsampleFraction, + FeatureFraction = param.FeatureFraction, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + }, + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + }; + + return context.BinaryClassification.Trainers.LightGbm(option); + } + } + + internal partial class LightGbmRegression + { + public override IEstimator BuildFromOption(MLContext context, LgbmOption param) + { + var option = new LightGbmRegressionTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfIterations = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + Booster = new GradientBooster.Options() + { + SubsampleFraction = param.SubsampleFraction, + FeatureFraction = param.FeatureFraction, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + }, + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + }; + + return context.Regression.Trainers.LightGbm(option); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs new file mode 100644 index 0000000000..36dbd5de7f --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class MapValueToKey + { + public override IEstimator BuildFromOption(MLContext context, MapValueToKeyOption param) + { + return context.Transforms.Conversion.MapValueToKey(param.OutputColumnName, param.InputColumnName); + } + } + + internal partial class MapKeyToValue + { + public override IEstimator BuildFromOption(MLContext context, MapKeyToValueOption param) + { + return context.Transforms.Conversion.MapKeyToValue(param.OutputColumnName, param.InputColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs new file mode 100644 index 0000000000..91900fd4e0 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class MatrixFactorization + { + public override IEstimator BuildFromOption(MLContext context, MatrixFactorizationOption param) + { + return context.Recommendation().Trainers.MatrixFactorization(param.LabelColumnName, param.MatrixColumnIndexColumnName, param.MatrixRowIndexColumnName, param.ApproximationRank, param.LearningRate, param.NumberOfIterations); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs new file mode 100644 index 0000000000..73fa584ffb --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class NormalizeMinMax + { + public override IEstimator BuildFromOption(MLContext context, NormalizeMinMaxOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.OutputColumnNames, param.InputColumnNames); + + return context.Transforms.NormalizeMinMax(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs new file mode 100644 index 0000000000..be5679ca3c --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class OneHotEncoding + { + public override IEstimator BuildFromOption(MLContext context, OneHotOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.Categorical.OneHotEncoding(inputOutputPairs); + } + } + + internal partial class OneHotHashEncoding + { + public override IEstimator BuildFromOption(MLContext context, OneHotOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.Categorical.OneHotHashEncoding(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs new file mode 100644 index 0000000000..e638798a88 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ReplaceMissingValues + { + public override IEstimator BuildFromOption(MLContext context, ReplaceMissingValueOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.ReplaceMissingValues(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs new file mode 100644 index 0000000000..1e073d56e4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class SdcaRegression + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaRegressionTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.Regression.Trainers.Sdca(option); + } + } + + internal partial class SdcaMaximumEntropyMulti + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaMaximumEntropyMulticlassTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.MulticlassClassification.Trainers.SdcaMaximumEntropy(option); + } + } + + internal partial class SdcaLogisticRegressionBinary + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaLogisticRegressionBinaryTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.BinaryClassification.Trainers.SdcaLogisticRegression(option); + } + } + + internal partial class SdcaLogisticRegressionOva + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaLogisticRegressionBinaryTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + var binaryTrainer = context.BinaryClassification.Trainers.SdcaLogisticRegression(option); + return context.MulticlassClassification.Trainers.OneVersusAll(binaryEstimator: binaryTrainer, labelColumnName: param.LabelColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs new file mode 100644 index 0000000000..f5e567baa0 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ConvertType + { + public override IEstimator BuildFromOption(MLContext context, ConvertTypeOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.Conversion.ConvertType(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs index db5634258d..d1e65bf898 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs @@ -50,11 +50,6 @@ public virtual IEstimator BuildFromOption(MLContext context, Param internal virtual IEnumerable NugetDependencies { get; } internal virtual string FunctionName { get; } - - internal virtual string ToDisplayString(Parameter param) - { - throw new NotImplementedException(); - } } internal abstract class SweepableEstimator : SweepableEstimator @@ -75,12 +70,5 @@ public override IEstimator BuildFromOption(MLContext context, Para { return this.BuildFromOption(context, param.AsType()); } - - internal abstract string ToDisplayString(TOption param); - - internal override string ToDisplayString(Parameter param) - { - return this.ToDisplayString(param.AsType()); - } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs index ceb30b1082..59d090db8a 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs @@ -54,52 +54,18 @@ namespace "); this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); this.Write(" { get; set; }\r\n\r\n"); } - this.Write(" public override IEnumerable CSharpUsingStatements \r\n {\r\n " + - " get => new string[] {"); + this.Write(" internal override IEnumerable CSharpUsingStatements \r\n {\r\n" + + " get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); - this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + - " {\r\n get => new string[] {"); + this.Write("};\r\n }\r\n\r\n internal override IEnumerable NugetDependencies\r" + + "\n {\r\n get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); - this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + - " get => \""); + this.Write("};\r\n }\r\n\r\n internal override string FunctionName \r\n {\r\n " + + " get => \""); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); this.Write("."); this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); - this.Write("\";\r\n }\r\n\r\n public override string ToCSharpCode()\r\n {\r\n"); -if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){ - this.Write(" return this.BuildCSharpCodeForTrainers();\r\n"); -}else if (Type == "OneVersusAll"){ - this.Write(" return this.BuildCSharpCodeForOva();\r\n"); -}else{ - this.Write(" return this.BuildCSharpCodeForTransformers();\r\n"); -} - this.Write(" }\r\n\r\n internal override void UpdatePropertiesFromOptions(Dictionar" + - "y options)\r\n {\r\n"); - foreach(var arg in ArgumentsList){ - var typeAttributeName = arg.ArgumentType; - var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); - if(typeAttributeName == "integer"){ - this.Write(" this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(" = options.ContainsKey(nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write("))? this.ToIntegerString(options[nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(")]) : this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(";\r\n "); -}else{ - this.Write(" this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(" = options.ContainsKey(nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write("))? options[nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(")] : this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(";\r\n "); -}} - this.Write(" }\r\n }\r\n}\r\n\r\n"); + this.Write("\";\r\n }\r\n }\r\n}\r\n\r\n"); return this.GenerationEnvironment.ToString(); } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt index b4e9a68558..fc0ccbb24f 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt @@ -30,43 +30,20 @@ namespace <#=NameSpace#> public string <#=propertyName#> { get; set; } <#}#> - public override IEnumerable CSharpUsingStatements + internal override IEnumerable CSharpUsingStatements { get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; } - public override IEnumerable NugetDependencies + internal override IEnumerable NugetDependencies { get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; } - public override string FunctionName + internal override string FunctionName { get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; } - - public override string ToCSharpCode() - { -<#if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){#> - return this.BuildCSharpCodeForTrainers(); -<#}else if (Type == "OneVersusAll"){#> - return this.BuildCSharpCodeForOva(); -<#}else{#> - return this.BuildCSharpCodeForTransformers(); -<#}#> - } - - internal override void UpdatePropertiesFromOptions(Dictionary options) - { -<# foreach(var arg in ArgumentsList){ - var typeAttributeName = arg.ArgumentType; - var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); - if(typeAttributeName == "integer"){#> - this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? this.ToIntegerString(options[nameof(this.<#=propertyName#>)]) : this.<#=propertyName#>; - <#}else{#> - this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? options[nameof(this.<#=propertyName#>)] : this.<#=propertyName#>; - <#}}#> - } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs index 9b1327aacb..5db7e76a39 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs @@ -48,24 +48,24 @@ namespace "); this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); this.Write(" defaultOption, SearchSpace<"); this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); - this.Write("> searchSpace = null)\r\n {\r\n this.Parameter = defaultOption;\r\n " + - " this.SearchSpace = searchSpace;\r\n this.EstimatorType = Esti" + - "matorType."); + this.Write("> searchSpace = null)\r\n {\r\n this.TParameter = defaultOption;\r\n " + + " this.SearchSpace = searchSpace;\r\n this.EstimatorType = Est" + + "imatorType."); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); this.Write(";\r\n }\r\n\r\n internal "); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); this.Write("()\r\n {\r\n this.EstimatorType = EstimatorType."); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); - this.Write(";\r\n this.Parameter = new "); + this.Write(";\r\n this.TParameter = new "); this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); - this.Write("();\r\n }\r\n \r\n public override IEnumerable CSharpUsingStat" + - "ements \r\n {\r\n get => new string[] {"); + this.Write("();\r\n }\r\n \r\n internal override IEnumerable CSharpUsingSt" + + "atements \r\n {\r\n get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); - this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + - " {\r\n get => new string[] {"); + this.Write("};\r\n }\r\n\r\n internal override IEnumerable NugetDependencies\r" + + "\n {\r\n get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); - this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + - " get => \""); + this.Write("};\r\n }\r\n\r\n internal override string FunctionName \r\n {\r\n " + + " get => \""); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); this.Write("."); this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt index ff6b3e5974..14ee696ea2 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt @@ -20,7 +20,7 @@ namespace <#=NameSpace#> { public <#=ClassName#>(<#=TOption#> defaultOption, SearchSpace<<#=TOption#>> searchSpace = null) { - this.Parameter = defaultOption; + this.TParameter = defaultOption; this.SearchSpace = searchSpace; this.EstimatorType = EstimatorType.<#=ClassName#>; } @@ -28,20 +28,20 @@ namespace <#=NameSpace#> internal <#=ClassName#>() { this.EstimatorType = EstimatorType.<#=ClassName#>; - this.Parameter = new <#=TOption#>(); + this.TParameter = new <#=TOption#>(); } - public override IEnumerable CSharpUsingStatements + internal override IEnumerable CSharpUsingStatements { get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; } - public override IEnumerable NugetDependencies + internal override IEnumerable NugetDependencies { get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; } - public override string FunctionName + internal override string FunctionName { get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; } From 365dd6a10e166a06ef1d32ff2c9ed3d6aeb51d26 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 6 Mar 2022 12:15:30 -0800 Subject: [PATCH 03/25] enable sweepabe estimator factory generator --- .../CodeGen/code_gen_flag.json | 2 +- ...Microsoft.ML.AutoML.SourceGenerator.csproj | 9 + .../SweepableEstimatorFactoryGenerator.cs | 55 +++ .../Template/SweepableEstimatorFactory.cs | 329 ++++++++++++++++++ .../Template/SweepableEstimatorFactory.tt | 36 ++ 5 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt diff --git a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json index cddb6933a5..6efed0e19a 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json +++ b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json @@ -1,6 +1,6 @@ { "EstimatorFactoryGenerator": false, - "CodeGenCatalogGenerator": false, + "SweepableEstimatorFactory": true, "EstimatorTypeGenerator": true, "SearchSpaceGenerator": true, "SweepableEstimatorGenerator": true diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj index df17513e14..ed8374d863 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj @@ -31,6 +31,11 @@ + + True + True + SweepableEstimatorFactory.tt + True True @@ -54,6 +59,10 @@ + + TextTemplatingFilePreprocessor + SweepableEstimatorFactory.cs + TextTemplatingFilePreprocessor EstimatorType.cs diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs new file mode 100644 index 0000000000..f8c5dba1f1 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; +using Microsoft.ML.AutoML.SourceGenerator.Template; + +namespace Microsoft.ML.AutoML.SourceGenerator +{ + [Generator] + public class SweepableEstimatorFactoryGenerator : ISourceGenerator + { + private const string className = "SweepableEstimatorFactory"; + + public void Execute(GeneratorExecutionContext context) + { + if (context.AdditionalFiles.Where(f => f.Path.Contains("code_gen_flag.json")).First() is AdditionalText text) + { + var json = text.GetText().ToString(); + var flags = JsonSerializer.Deserialize>(json); + if (flags.TryGetValue(nameof(SweepableEstimatorFactoryGenerator), out var res) && res == false) + { + return; + } + } + + var trainers = context.AdditionalFiles.Where(f => f.Path.Contains("trainer-estimators.json")) + .SelectMany(file => Utils.GetEstimatorsFromJson(file.GetText().ToString()).Estimators, (text, estimator) => (estimator.FunctionName, estimator.EstimatorTypes, estimator.SearchOption)) + .SelectMany(union => union.EstimatorTypes.Select(t => (Utils.CreateEstimatorName(union.FunctionName, t), Utils.ToTitleCase(union.SearchOption)))) + .ToArray(); + + var transformers = context.AdditionalFiles.Where(f => f.Path.Contains("transformer-estimators.json")) + .SelectMany(file => Utils.GetEstimatorsFromJson(file.GetText().ToString()).Estimators, (text, estimator) => (estimator.FunctionName, estimator.EstimatorTypes, estimator.SearchOption)) + .SelectMany(union => union.EstimatorTypes.Select(t => (Utils.CreateEstimatorName(union.FunctionName, t), Utils.ToTitleCase(union.SearchOption)))) + .ToArray(); + + var code = new SweepableEstimatorFactory() + { + NameSpace = Constant.CodeGeneratorNameSpace, + EstimatorNames = trainers.Concat(transformers), + }; + + context.AddSource(className + ".cs", SourceText.From(code.TransformText(), Encoding.UTF8)); + } + + public void Initialize(GeneratorInitializationContext context) + { + } + } +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs new file mode 100644 index 0000000000..649537e3bf --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs @@ -0,0 +1,329 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version: 17.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +namespace Microsoft.ML.AutoML.SourceGenerator.Template +{ + using System.Linq; + using System.Text; + using System.Collections.Generic; + using System; + + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class SweepableEstimatorFactory : SweepableEstimatorFactoryBase + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write("\r\nusing System;\r\nusing System.Collections.Generic;\r\nusing System.Text;\r\nusing New" + + "tonsoft.Json;\r\nusing Newtonsoft.Json.Linq;\r\nusing Microsoft.ML.SearchSpace;\r\nusi" + + "ng Microsoft.ML;\r\n\r\nnamespace "); + this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); + this.Write("\r\n{\r\n internal static class SweepableEsitmatorFactory\r\n {\r\n"); + foreach((var estimator, var tOption) in EstimatorNames){ + this.Write(" public static "); + this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); + this.Write(" Create"); + this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); + this.Write("("); + this.Write(this.ToStringHelper.ToStringWithCulture(tOption)); + this.Write(" defaultOption, SearchSpace<"); + this.Write(this.ToStringHelper.ToStringWithCulture(tOption)); + this.Write("> searchSpace = null)\r\n {\r\n if(searchSpace == null){\r\n " + + " searchSpace = new SearchSpace<"); + this.Write(this.ToStringHelper.ToStringWithCulture(tOption)); + this.Write(">(defaultOption);\r\n }\r\n\r\n return new "); + this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); + this.Write("(defaultOption, searchSpace);\r\n }\r\n\r\n"); +} + this.Write(" }\r\n}\r\n\r\n"); + return this.GenerationEnvironment.ToString(); + } + +public string NameSpace {get;set;} +public IEnumerable<(string, string)> EstimatorNames {get;set;} + + } + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class SweepableEstimatorFactoryBase + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + protected System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; + } + set + { + this.generationEnvironmentField = value; + } + } + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + } + return this.errorsField; + } + } + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); + } + return this.indentLengthsField; + } + } + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; + } + } + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; + } + set + { + this.sessionField = value; + } + } + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } + } + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; + } + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); + } + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); + } + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); + } + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; + } + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; + } + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; + } + set + { + if ((value != null)) + { + this.formatProviderField = value; + } + } + } + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { + throw new global::System.ArgumentNullException("objectToConvert"); + } + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); + } + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); + } + } + } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion + } + #endregion +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt new file mode 100644 index 0000000000..bbf6298711 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt @@ -0,0 +1,36 @@ +<#@ template language="C#" linePragmas="false" visibility = "internal" #> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> + +using System; +using System.Collections.Generic; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Microsoft.ML.SearchSpace; +using Microsoft.ML; + +namespace <#=NameSpace#> +{ + internal static class SweepableEsitmatorFactory + { +<# foreach((var estimator, var tOption) in EstimatorNames){#> + public static <#=estimator#> Create<#=estimator#>(<#=tOption#> defaultOption, SearchSpace<<#=tOption#>> searchSpace = null) + { + if(searchSpace == null){ + searchSpace = new SearchSpace<<#=tOption#>>(defaultOption); + } + + return new <#=estimator#>(defaultOption, searchSpace); + } + +<#}#> + } +} + +<#+ +public string NameSpace {get;set;} +public IEnumerable<(string, string)> EstimatorNames {get;set;} +#> From 50d2c286a3d5f55e1fd84666772dabf5fb8876f6 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 11:16:09 -0700 Subject: [PATCH 04/25] add tests for sweepable estimator pipeline and multi-model pipeline --- .../SweepableEstimatorPipelineTest.cs | 90 ++++++++++++++++++- .../Template/SweepableEstimatorFactory.cs | 2 +- .../Template/SweepableEstimatorFactory.tt | 2 +- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 2b67e1d996..76195bdf7d 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -11,17 +11,24 @@ using ApprovalTests.Reporters; using FluentAssertions; using Microsoft.ML.TestFramework; -using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; +using Microsoft.ML.AutoML.CodeGen; +using System.Text.Json; namespace Microsoft.ML.AutoML.Test { public class SweepableEstimatorPipelineTest : BaseTestClass { + private readonly JsonSerializerOptions _jsonSerializerOptions; + public SweepableEstimatorPipelineTest(ITestOutputHelper output) : base(output) { + this._jsonSerializerOptions = new JsonSerializerOptions() + { + WriteIndented = true, + }; } [Fact] @@ -51,5 +58,86 @@ public void MultiModelPipeline_append_test() pipeline.BuildSweepableEstimatorPipeline("e0 * e2").ToString().Should().Be("Concatenate=>ApplyOnnxModel"); pipeline.BuildSweepableEstimatorPipeline("e1 * Nil").ToString().Should().Be("ConvertType"); } + + [Fact] + public void MultiModelPipeline_append_pipeline_test() + { + var e1 = new SweepableEstimator(CodeGen.EstimatorType.Concatenate); + var e2 = new SweepableEstimator(CodeGen.EstimatorType.ConvertType); + var e3 = new SweepableEstimator(CodeGen.EstimatorType.ApplyOnnxModel); + var e4 = new SweepableEstimator(CodeGen.EstimatorType.LightGbmBinary); + var e5 = new SweepableEstimator(CodeGen.EstimatorType.FastTreeBinary); + + var pipeline1 = new MultiModelPipeline(); + var pipeline2 = new MultiModelPipeline(); + + pipeline1 = pipeline1.Append(e1 + e2 * e3); + pipeline2 = pipeline2.Append(e1 * (e3 + e4) + e5); + + pipeline1 = pipeline1.Append(pipeline2); + + pipeline1.Schema.ToString().Should().Be("(e0 + e1 * e2) * (e3 * (e4 + e5) + e6)"); + } + + [Fact] + public void SweepableEstimatorPipeline_search_space_test() + { + var pipeline = this.CreateSweepbaleEstimatorPipeline(); + pipeline.SearchSpace.FeatureSpaceDim.Should().Be(15); + + // TODO + // verify other properties in search space. + } + + [Fact] + public void SweepableEstimatorPipeline_can_be_created_from_MultiModelPipeline() + { + var multiModelPipeline = this.CreateMultiModelPipeline(); + var pipelines = multiModelPipeline.PipelineIds; + + pipelines.Should().BeEquivalentTo("e0 * e3 * e4", "e1 * e2 * e3 * e4", "e0 * Nil * e4", "e1 * e2 * Nil * e4", "Nil * e3 * e4", "e0 * e3 * e5", "e1 * e2 * e3 * e5", "e0 * Nil * e5", "e1 * e2 * Nil * e5", "Nil * e3 * e5", "Nil * Nil * e4", "Nil * Nil * e5"); + var singleModelPipeline = multiModelPipeline.BuildSweepableEstimatorPipeline(pipelines[0]); + singleModelPipeline.ToString().Should().Be("ReplaceMissingValues=>Concatenate=>LightGbmBinary"); + singleModelPipeline = multiModelPipeline.BuildSweepableEstimatorPipeline(pipelines[2]); + singleModelPipeline.ToString().Should().Be("ReplaceMissingValues=>LightGbmBinary"); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void SweepableEstimatorPipeline_search_space_init_value_test() + { + var singleModelPipeline = this.CreateSweepbaleEstimatorPipeline(); + var defaultParam = singleModelPipeline.SearchSpace.SampleFromFeatureSpace(singleModelPipeline.SearchSpace.Default); + Approvals.Verify(JsonSerializer.Serialize(defaultParam, this._jsonSerializerOptions)); + } + + private SweepableEstimatorPipeline CreateSweepbaleEstimatorPipeline() + { + var concat = SweepableEstimatorFactory.CreateConcatenate(new ConcatOption()); + var replaceMissingValue = SweepableEstimatorFactory.CreateReplaceMissingValues(new ReplaceMissingValueOption()); + var oneHot = SweepableEstimatorFactory.CreateOneHotEncoding(new OneHotOption()); + var lightGbm = SweepableEstimatorFactory.CreateLightGbmBinary(new LgbmOption()); + var fastTree = SweepableEstimatorFactory.CreateFastTreeBinary(new FastTreeOption()); + + var pipeline = new SweepableEstimatorPipeline(new SweepableEstimator[] { concat, replaceMissingValue, oneHot, lightGbm, fastTree }); + return pipeline; + } + + private MultiModelPipeline CreateMultiModelPipeline() + { + var concat = SweepableEstimatorFactory.CreateConcatenate(new ConcatOption()); + var replaceMissingValue = SweepableEstimatorFactory.CreateReplaceMissingValues(new ReplaceMissingValueOption()); + var oneHot = SweepableEstimatorFactory.CreateOneHotEncoding(new OneHotOption()); + var lightGbm = SweepableEstimatorFactory.CreateLightGbmBinary(new LgbmOption()); + var fastTree = SweepableEstimatorFactory.CreateFastTreeBinary(new FastTreeOption()); + + var pipeline = new MultiModelPipeline(); + pipeline = pipeline.AppendOrSkip(replaceMissingValue + replaceMissingValue * oneHot); + pipeline = pipeline.AppendOrSkip(concat); + pipeline = pipeline.Append(lightGbm + fastTree); + + return pipeline; + } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs index 649537e3bf..820d931f14 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs @@ -29,7 +29,7 @@ public virtual string TransformText() "tonsoft.Json;\r\nusing Newtonsoft.Json.Linq;\r\nusing Microsoft.ML.SearchSpace;\r\nusi" + "ng Microsoft.ML;\r\n\r\nnamespace "); this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); - this.Write("\r\n{\r\n internal static class SweepableEsitmatorFactory\r\n {\r\n"); + this.Write("\r\n{\r\n internal static class SweepableEstimatorFactory\r\n {\r\n"); foreach((var estimator, var tOption) in EstimatorNames){ this.Write(" public static "); this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt index bbf6298711..c9fdee1607 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt @@ -14,7 +14,7 @@ using Microsoft.ML; namespace <#=NameSpace#> { - internal static class SweepableEsitmatorFactory + internal static class SweepableEstimatorFactory { <# foreach((var estimator, var tOption) in EstimatorNames){#> public static <#=estimator#> Create<#=estimator#>(<#=tOption#> defaultOption, SearchSpace<<#=tOption#>> searchSpace = null) From 8682502043ef6ad2db1e001b687e673028f2bab8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 14:15:00 -0700 Subject: [PATCH 05/25] add sweepable extension tests and more tests --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 169 ++++++++++++++++++ .../API/SweepableExtension.cs | 30 +++- .../Converter/MultiModelPipelineConverter.cs | 34 ++++ .../Converter/SweepableEstimatorConverter.cs | 37 ++++ .../SweepableEstimatorPipelineConverter.cs | 39 ++++ .../SweepableEstimator/Estimator.cs | 1 + .../SweepableEstimator/MultiModelPipeline.cs | 2 + .../SweepableEstimator/SweepableEstimator.cs | 2 + .../SweepableEstimatorPipeline.cs | 2 + ...delPipeline_search_space_test.approved.txt | 0 ...delPipeline_search_space_test.received.txt | 61 +++++++ ..._search_space_init_value_test.approved.txt | 28 +++ ...EstimatorAndBinaryClassifiers.approved.txt | 66 +++++++ ...IEstimatorAndMultiClassifiers.approved.txt | 84 +++++++++ ...neFromIEstimatorAndRegressors.approved.txt | 84 +++++++++ ...eEstimatorAndMultiClassifiers.approved.txt | 90 ++++++++++ ...eEstimatorAndMultiClassifiers.received.txt | 90 ++++++++++ ...orPipelineAndMultiClassifiers.approved.txt | 88 +++++++++ ...orPipelineAndMultiClassifiers.received.txt | 88 +++++++++ .../SweepableExtensionTest.cs | 147 +++++++++++++++ .../Template/EstimatorType.cs | 6 +- .../Template/EstimatorType.tt | 1 + 22 files changed, 1145 insertions(+), 4 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index b704188f68..3b822b647f 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -3,6 +3,10 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.Contracts; +using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.Data; using Microsoft.ML.SearchSpace; @@ -289,5 +293,170 @@ internal SweepableEstimator CreateSweepableEstimator(Func factory(context, param.AsType()), ss); } + + internal SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, + SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) + { + var res = new List(); + + if (useFastTree) + { + fastTreeOption = fastTreeOption ?? new FastTreeOption(); + fastTreeOption.LabelColumnName = labelColumnName; + fastTreeOption.FeatureColumnName = featureColumnName; + fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + } + + if (useFastForest) + { + fastForestOption = fastForestOption ?? new FastForestOption(); + fastForestOption.LabelColumnName = labelColumnName; + fastForestOption.FeatureColumnName = featureColumnName; + fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + } + + if (useLgbm) + { + lgbmOption = lgbmOption ?? new LgbmOption(); + lgbmOption.LabelColumnName = labelColumnName; + lgbmOption.FeatureColumnName = featureColumnName; + lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + } + + if (useLbfgs) + { + lbfgsOption = lbfgsOption ?? new LbfgsOption(); + lbfgsOption.LabelColumnName = labelColumnName; + lbfgsOption.FeatureColumnName = featureColumnName; + lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + } + + if (useSdca) + { + sdcaOption = sdcaOption ?? new SdcaOption(); + sdcaOption.LabelColumnName = labelColumnName; + sdcaOption.FeatureColumnName = featureColumnName; + sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + } + + return res.ToArray(); + } + + internal SweepableEstimator[] MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, + SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) + { + var res = new List(); + + if (useFastTree) + { + fastTreeOption = fastTreeOption ?? new FastTreeOption(); + fastTreeOption.LabelColumnName = labelColumnName; + fastTreeOption.FeatureColumnName = featureColumnName; + fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + } + + if (useFastForest) + { + fastForestOption = fastForestOption ?? new FastForestOption(); + fastForestOption.LabelColumnName = labelColumnName; + fastForestOption.FeatureColumnName = featureColumnName; + fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + } + + if (useLgbm) + { + lgbmOption = lgbmOption ?? new LgbmOption(); + lgbmOption.LabelColumnName = labelColumnName; + lgbmOption.FeatureColumnName = featureColumnName; + lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + } + + if (useLbfgs) + { + lbfgsOption = lbfgsOption ?? new LbfgsOption(); + lbfgsOption.LabelColumnName = labelColumnName; + lbfgsOption.FeatureColumnName = featureColumnName; + lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + } + + if (useSdca) + { + sdcaOption = sdcaOption ?? new SdcaOption(); + sdcaOption.LabelColumnName = labelColumnName; + sdcaOption.FeatureColumnName = featureColumnName; + sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + } + + return res.ToArray(); + } + + internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, + SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) + { + var res = new List(); + + if (useFastTree) + { + fastTreeOption = fastTreeOption ?? new FastTreeOption(); + fastTreeOption.LabelColumnName = labelColumnName; + fastTreeOption.FeatureColumnName = featureColumnName; + fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeTweedieRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + } + + if (useFastForest) + { + fastForestOption = fastForestOption ?? new FastForestOption(); + fastForestOption.LabelColumnName = labelColumnName; + fastForestOption.FeatureColumnName = featureColumnName; + fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + } + + if (useLgbm) + { + lgbmOption = lgbmOption ?? new LgbmOption(); + lgbmOption.LabelColumnName = labelColumnName; + lgbmOption.FeatureColumnName = featureColumnName; + lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + } + + if (useLbfgs) + { + lbfgsOption = lbfgsOption ?? new LbfgsOption(); + lbfgsOption.LabelColumnName = labelColumnName; + lbfgsOption.FeatureColumnName = featureColumnName; + lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + } + + if (useSdca) + { + sdcaOption = sdcaOption ?? new SdcaOption(); + sdcaOption.LabelColumnName = labelColumnName; + sdcaOption.FeatureColumnName = featureColumnName; + sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + } + + return res.ToArray(); + } } } diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index baf72267ce..1848d8c128 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; namespace Microsoft.ML.AutoML @@ -27,7 +28,34 @@ public static SweepableEstimatorPipeline Append(this SweepableEstimator estimato public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, IEstimator estimator1) { - return estimator.Append(estimator1); + return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); + } + + public static MultiModelPipeline Append(this IEstimator estimator, params SweepableEstimator[] estimators) + { + var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); + var multiModelPipeline = new MultiModelPipeline().Append(sweepableEstimator).Append(estimators); + + return multiModelPipeline; + } + + public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) + { + var multiModelPipeline = new MultiModelPipeline(); + foreach (var estimator in pipeline.Estimators) + { + multiModelPipeline = multiModelPipeline.Append(estimator); + } + + return multiModelPipeline.Append(estimators); + } + + public static MultiModelPipeline Append(this SweepableEstimator estimator, params SweepableEstimator[] estimators) + { + var multiModelPipeline = new MultiModelPipeline(); + multiModelPipeline = multiModelPipeline.Append(estimator); + + return multiModelPipeline.Append(estimators); } } } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs new file mode 100644 index 0000000000..d3107c939a --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Microsoft.ML.AutoML +{ + internal class MultiModelPipelineConverter : JsonConverter + { + public override MultiModelPipeline Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jValue = JsonValue.Parse(ref reader); + var schema = jValue["schema"].GetValue(); + var estimators = jValue["estimator"].GetValue>(); + + return new MultiModelPipeline(estimators, Entity.FromExpression(schema)); + } + + public override void Write(Utf8JsonWriter writer, MultiModelPipeline value, JsonSerializerOptions options) + { + var jsonObject = JsonNode.Parse("{}"); + jsonObject["schema"] = value.Schema.ToString(); + jsonObject["estimators"] = JsonValue.Create(value.Estimators); + + jsonObject.WriteTo(writer, options); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs new file mode 100644 index 0000000000..362de6eaa4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.ML.AutoML.CodeGen; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class SweepableEstimatorConverter : JsonConverter + { + public override SweepableEstimator Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jsonObject = JsonValue.Parse(ref reader); + var estimatorType = jsonObject["estimatorType"].GetValue(); + var parameter = jsonObject["parameter"].GetValue(); + var estimator = new SweepableEstimator(estimatorType); + estimator.Parameter = parameter; + + return estimator; + } + + public override void Write(Utf8JsonWriter writer, SweepableEstimator value, JsonSerializerOptions options) + { + var jObject = JsonObject.Parse("{}"); + jObject["estimatorType"] = JsonValue.Create(value.EstimatorType); + jObject["parameter"] = JsonValue.Create(value.Parameter); + jObject.WriteTo(writer, options); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs new file mode 100644 index 0000000000..403654ea21 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.ML.SearchSpace; +using Newtonsoft.Json; + +namespace Microsoft.ML.AutoML +{ + internal class SweepableEstimatorPipelineConverter : JsonConverter + { + public override SweepableEstimatorPipeline Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jNode = JsonNode.Parse(ref reader); + var parameter = jNode["parameter"].GetValue(); + var estimators = jNode["estimators"].GetValue(); + var pipeline = new SweepableEstimatorPipeline(estimators, parameter); + + return pipeline; + } + + public override void Write(Utf8JsonWriter writer, SweepableEstimatorPipeline value, JsonSerializerOptions options) + { + var parameter = value.Parameter; + var estimators = value.Estimators; + var jNode = JsonNode.Parse("{}"); + jNode["parameter"] = JsonValue.Create(parameter); + jNode["estimators"] = JsonValue.Create(estimators); + + jNode.WriteTo(writer, options); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs index d26fa97134..11339ede14 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs @@ -12,6 +12,7 @@ internal class Estimator protected Estimator() { this.Parameter = Parameter.CreateNestedParameter(); + this.EstimatorType = EstimatorType.Unknown; } internal Estimator(EstimatorType estimatorType) diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs index 55777bc9ba..3be0a7dff8 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs @@ -6,9 +6,11 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Text.Json.Serialization; namespace Microsoft.ML.AutoML { + [JsonConverter(typeof(MultiModelPipelineConverter))] internal class MultiModelPipeline { private static readonly StringEntity _nilStringEntity = new StringEntity("Nil"); diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs index d1e65bf898..3dc3186978 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Text.Json.Serialization; using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.SearchSpace; @@ -12,6 +13,7 @@ namespace Microsoft.ML.AutoML /// /// Estimator with search space. /// + [JsonConverter(typeof(SweepableEstimatorConverter))] internal class SweepableEstimator : Estimator { private readonly Func> _factory; diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs index 5aae62e2d1..1f1909638d 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs @@ -5,11 +5,13 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Serialization; using Microsoft.ML.Data; using Microsoft.ML.SearchSpace; namespace Microsoft.ML.AutoML { + [JsonConverter(typeof(SweepableEstimatorPipelineConverter))] internal class SweepableEstimatorPipeline { private readonly List _estimators; diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt new file mode 100644 index 0000000000..7d3e8e1861 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt @@ -0,0 +1,61 @@ +{ + "0": { + "FeatureSpaceDim": 0, + "Default": [], + "Step": [] + }, + "1": { + "FeatureSpaceDim": 0, + "Default": [], + "Step": [] + }, + "2": { + "FeatureSpaceDim": 0, + "Default": [], + "Step": [] + }, + "3": { + "FeatureSpaceDim": 9, + "Default": [ + 1, + 0, + 1, + 1, + 0.71428571428571408, + 0, + 0, + 0, + 1 + ], + "Step": [ + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "4": { + "FeatureSpaceDim": 6, + "Default": [ + 1, + 0.89689626841273451, + 0.71428571428571408, + 0.55365468248122718, + 0, + 0 + ], + "Step": [ + null, + null, + null, + null, + null, + null + ] + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt new file mode 100644 index 0000000000..e4b6191616 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt @@ -0,0 +1,28 @@ +{ + "0": {}, + "1": {}, + "2": {}, + "3": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 255, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + }, + "4": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 255, + "FeatureFraction": 1, + "LearningRate": 0.099999999999999978, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt new file mode 100644 index 0000000000..fa86919915 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt @@ -0,0 +1,66 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeBinary", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmBinary", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionBinary", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "SdcaLogisticRegressionBinary", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..00a3d413c7 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,84 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt new file mode 100644 index 0000000000..00a3d413c7 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt @@ -0,0 +1,84 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..d6f4d7d027 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,90 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt new file mode 100644 index 0000000000..ada341d69d --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt @@ -0,0 +1,90 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..21058f9cbd --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt @@ -0,0 +1,88 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt new file mode 100644 index 0000000000..adcaa9ec62 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt @@ -0,0 +1,88 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs new file mode 100644 index 0000000000..72df8a0dec --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -0,0 +1,147 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; +using Microsoft.ML.AutoML.CodeGen; +using FluentAssertions; +using Microsoft.ML.TestFramework; +using Xunit.Abstractions; +using ApprovalTests.Namers; +using ApprovalTests.Reporters; +using System.Text.Json.Serialization; +using System.Text.Json; +using ApprovalTests; + +namespace Microsoft.ML.AutoML.Test +{ + public class SweepableExtensionTest : BaseTestClass + { + private readonly JsonSerializerOptions _jsonSerializerOptions; + + public SweepableExtensionTest(ITestOutputHelper output) + : base(output) + { + this._jsonSerializerOptions = new JsonSerializerOptions() + { + WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(), + }, + }; + + this._jsonSerializerOptions.Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping; + } + + [Fact] + public void CreateSweepableEstimatorPipelineFromIEstimatorTest() + { + var context = new MLContext(); + var estimator = context.Transforms.Concatenate("output", "input"); + var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + + pipeline.ToString().Should().Be("Unknown=>FastForestBinary"); + } + + [Fact] + public void AppendIEstimatorToSweepabeEstimatorPipelineTest() + { + var context = new MLContext(); + var estimator = context.Transforms.Concatenate("output", "input"); + var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); + + pipeline.ToString().Should().Be("Unknown=>FastForestBinary=>Unknown"); + } + + [Fact] + public void CreateSweepableEstimatorPipelineFromSweepableEstimatorTest() + { + var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); + var pipeline = estimator.Append(estimator); + + pipeline.ToString().Should().Be("FastForestBinary=>FastForestBinary"); + } + + [Fact] + public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorTest() + { + var context = new MLContext(); + var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); + var pipeline = estimator.Append(context.Transforms.Concatenate("output", "input")); + + pipeline.ToString().Should().Be("FastForestBinary=>Unknown"); + + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().BinaryClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromIEstimatorAndRegressors() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() + { + var context = new MLContext(); + var pipeline = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()) + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption())) + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + } +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs index 7e1c51100a..f0f9f3a960 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs @@ -40,9 +40,9 @@ public virtual string TransformText() this.Write(this.ToStringHelper.ToStringWithCulture(e)); this.Write(",\r\n"); } - this.Write(" }\r\n\r\n public static class EstimatorTypeExtension\r\n {\r\n public st" + - "atic bool IsTrainer(this EstimatorType estimatorType)\r\n {\r\n sw" + - "itch(estimatorType)\r\n {\r\n"); + this.Write(" Unknown,\r\n }\r\n\r\n public static class EstimatorTypeExtension\r\n {\r" + + "\n public static bool IsTrainer(this EstimatorType estimatorType)\r\n " + + " {\r\n switch(estimatorType)\r\n {\r\n"); foreach(var estimator in TrainerNames){ this.Write(" case EstimatorType."); this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt index d95e7a3066..0fcb75f298 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt @@ -18,6 +18,7 @@ namespace <#=NameSpace#> <# foreach(var e in TransformerNames){#> <#=e#>, <#}#> + Unknown, } public static class EstimatorTypeExtension From e5ac469fc5b8ec82f62ab3707a1494b68dfe5277 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 20:53:36 -0700 Subject: [PATCH 06/25] add automl experiment --- .../AutoMLExperiment/AutoMLExperiment.cs | 192 +++++++++++++ .../AutoMLExperiment/IDatasetSettings.cs | 28 ++ .../AutoMLExperiment/IMetricSettings.cs | 66 +++++ .../AutoMLExperiment/IMonitor.cs | 58 ++++ .../AutoMLExperiment/TrialResult.cs | 21 ++ .../AutoMLExperiment/TrialRunner.cs | 99 +++++++ .../AutoMLExperiment/TrialSettings.cs | 24 ++ .../HyperParameterProposer.cs | 43 +++ .../ITrialSettingsProposer.cs | 20 ++ .../TrialSettingsProposer/PipelineProposer.cs | 269 ++++++++++++++++++ src/Microsoft.ML.AutoML/Utils/ArrayMath.cs | 168 +++++++++++ src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs | 17 ++ 12 files changed, 1005 insertions(+) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs create mode 100644 src/Microsoft.ML.AutoML/Utils/ArrayMath.cs create mode 100644 src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs new file mode 100644 index 0000000000..c519f76360 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -0,0 +1,192 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class AutoMLExperiment + { + private readonly AutoMLExperimentSettings _settings; + private readonly MLContext _context; + private double _bestError = double.MaxValue; + private TrialResult _bestTrialResult = null; + + public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) + { + this._context = context; + this._settings = settings; + } + + public AutoMLExperiment SetTrainingTimeInSeconds(int trainingTimeInSeconds) + { + this._settings.MaxExperimentTimeInSeconds = (uint)trainingTimeInSeconds; + return this; + } + + public AutoMLExperiment SetDataset(IDataView train, IDataView test) + { + this._settings.DatasetSettings = new TrainTestDatasetSettings() + { + TrainDataset = train, + TestDataset = test + }; + + return this; + } + + public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) + { + this._settings.DatasetSettings = new CrossValidateDatasetSettings() + { + Dataset = dataset, + Fold = fold, + }; + + return this; + } + + public AutoMLExperiment SetTunerFactory(Func tunerFactory) + { + this._settings.TunerFactory = tunerFactory; + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") + { + this._settings.EvaluateMetric = new BinaryMetricSettings() + { + Metric = metric, + PredictedColumn = predictedColumn, + TruthColumn = truthColumn, + }; + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") + { + this._settings.EvaluateMetric = new MultiClassMetricSettings() + { + Metric = metric, + PredictedColumn = predictedColumn, + TruthColumn = truthColumn, + }; + + return this; + } + + public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") + { + this._settings.EvaluateMetric = new RegressionMetricSettings() + { + Metric = metric, + PredictedColumn = predictedColumn, + TruthColumn = truthColumn, + }; + + return this; + } + + /// + /// Run experiment and return the best trial result. + /// + /// + public Task Run() + { + this.ValidateSettings(); + var cts = new CancellationTokenSource(); + cts.CancelAfter((int)this._settings.MaxExperimentTimeInSeconds * 1000); + this._settings.CancellationToken.Register(() => cts.Cancel()); + + return this.RunAsync(cts.Token); + } + + private async Task RunAsync(CancellationToken ct) + { + var trialNum = 0; + var pipelineProposer = new PipelineProposer(this._settings.Seed ?? 0); + var hyperParameterProposer = new HyperParameterProposer(); + + while (true) + { + if (ct.IsCancellationRequested) + { + break; + } + + var setting = new TrialSettings() + { + ExperimentSettings = this._settings, + TrialId = trialNum++, + }; + + setting = pipelineProposer.Propose(setting); + setting = hyperParameterProposer.Propose(setting); + + ITrialRunner runner = (this._settings.DatasetSettings, this._settings.EvaluateMetric) switch + { + (CrossValidateDatasetSettings, BinaryMetricSettings) => new BinaryClassificationCVRunner(), + (TrainTestDatasetSettings, BinaryMetricSettings) => new BinaryClassificationTrainTestRunner(), + _ => throw new NotImplementedException(), + }; + + this._settings.Monitor.ReportRunningTrial(setting); + var trialResult = runner.Run(this._context, setting); + this._settings.Monitor.ReportCompletedTrial(trialResult); + hyperParameterProposer.Update(setting, trialResult); + pipelineProposer.Update(setting, trialResult); + + var error = this._settings.EvaluateMetric.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < this._bestError) + { + this._bestTrialResult = trialResult; + this._bestError = error; + this._settings.Monitor.ReportBestTrial(trialResult); + } + } + + if (this._bestTrialResult == null) + { + throw new TimeoutException("Training time finished without completing a trial run"); + } + else + { + return await Task.FromResult(this._bestTrialResult); + } + } + + private void ValidateSettings() + { + Contracts.Assert(this._settings.MaxExperimentTimeInSeconds > 0, $"{nameof(ExperimentSettings.MaxExperimentTimeInSeconds)} must be larger than 0"); + Contracts.Assert(this._settings.DatasetSettings != null, $"{nameof(this._settings.DatasetSettings)} must be not null"); + Contracts.Assert(this._settings.TunerFactory != null, $"{nameof(this._settings.TunerFactory)} must be not null"); + Contracts.Assert(this._settings.EvaluateMetric != null, $"{nameof(this._settings.EvaluateMetric)} must be not null"); + } + + + public class AutoMLExperimentSettings : ExperimentSettings + { + public IDatasetSettings DatasetSettings { get; set; } + + public IMetricSettings EvaluateMetric { get; set; } + + public MultiModelPipeline Pipeline { get; set; } + + public Func TunerFactory { get; set; } + + public IMonitor Monitor { get; set; } + + public int? Seed { get; set; } + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs new file mode 100644 index 0000000000..023a35ecc6 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IDatasetSettings.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML +{ + internal interface IDatasetSettings + { + } + + internal class TrainTestDatasetSettings : IDatasetSettings + { + public IDataView TrainDataset { get; set; } + + public IDataView TestDataset { get; set; } + } + + internal class CrossValidateDatasetSettings : IDatasetSettings + { + public IDataView Dataset { get; set; } + + public int? Fold { get; set; } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs new file mode 100644 index 0000000000..8a23c6886e --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.Data; + +namespace Microsoft.ML.AutoML +{ + internal interface IMetricSettings + { + bool IsMaximize { get; } + } + + internal class BinaryMetricSettings : IMetricSettings + { + public BinaryClassificationMetric Metric { get; set; } + + public string PredictedColumn { get; set; } + + public string TruthColumn { get; set; } + + public bool IsMaximize => this.Metric switch + { + BinaryClassificationMetric.Accuracy => true, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => true, + BinaryClassificationMetric.AreaUnderRocCurve => true, + BinaryClassificationMetric.PositivePrecision => true, + BinaryClassificationMetric.NegativePrecision => true, + BinaryClassificationMetric.NegativeRecall => true, + BinaryClassificationMetric.PositiveRecall => true, + BinaryClassificationMetric.F1Score => throw new NotImplementedException(), + _ => throw new NotImplementedException(), + }; + } + + internal class MultiClassMetricSettings : IMetricSettings + { + public MulticlassClassificationMetric Metric { get; set; } + + public string PredictedColumn { get; set; } + + public string TruthColumn { get; set; } + + public bool IsMaximize => this.Metric switch + { + _ => throw new NotImplementedException(), + }; + } + + internal class RegressionMetricSettings : IMetricSettings + { + public RegressionMetric Metric { get; set; } + + public string PredictedColumn { get; set; } + + public string TruthColumn { get; set; } + + public bool IsMaximize => this.Metric switch + { + _ => throw new NotImplementedException(), + }; + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs new file mode 100644 index 0000000000..3ac894ac78 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.AutoML +{ + internal interface IMonitor + { + void ReportCompletedTrial(TrialResult result); + + void ReportBestTrial(TrialResult result); + + void ReportFailTrial(TrialResult result); + + void ReportRunningTrial(TrialSettings setting); + } + + // this monitor redirects output result to context.log + internal class MLContextMonitor : IMonitor + { + private readonly MLContext _context; + private readonly IChannel _logger; + private readonly List _completedTrials; + + public MLContextMonitor(MLContext context) + { + this._context = context; + this._logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); + this._completedTrials = new List(); + } + + public void ReportBestTrial(TrialResult result) + { + this._logger.Info($"Update Best Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + } + + public void ReportCompletedTrial(TrialResult result) + { + this._logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + this._completedTrials.Add(result); + } + + public void ReportFailTrial(TrialResult result) + { + this._logger.Info($"Update Failed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + } + + public void ReportRunningTrial(TrialSettings setting) + { + this._logger.Info($"Update Running Trial - Id: {setting.TrialId} - Pipeline: {setting.Pipeline}"); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs new file mode 100644 index 0000000000..97a93df44b --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML +{ + internal class TrialResult + { + public TrialSettings TrialSettings { get; set; } + + public ITransformer Model { get; set; } + + public double Metric { get; set; } + + public float DurationInMilliseconds { get; set; } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs new file mode 100644 index 0000000000..6cfa7c4781 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics; + +namespace Microsoft.ML.AutoML +{ + internal interface ITrialRunner + { + TrialResult Run(MLContext context, TrialSettings settings); + } + + internal class BinaryClassificationCVRunner : ITrialRunner + { + public TrialResult Run(MLContext context, TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + + var pipeline = settings.Pipeline.BuildTrainingPipeline(context, settings.Parameter); + var metrics = context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + BinaryClassificationMetric.PositivePrecision => res.Metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => res.Metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => res.Metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => res.Metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class BinaryClassificationTrainTestRunner : ITrialRunner + { + public TrialResult Run(MLContext context, TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is BinaryMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var pipeline = settings.Pipeline.BuildTrainingPipeline(context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.PredictedColumn, predictedLabelColumnName: metricSettings.TruthColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var metric = metricSettings.Metric switch + { + BinaryClassificationMetric.PositivePrecision => metrics.PositivePrecision, + BinaryClassificationMetric.Accuracy => metrics.Accuracy, + BinaryClassificationMetric.AreaUnderRocCurve => metrics.AreaUnderRocCurve, + BinaryClassificationMetric.AreaUnderPrecisionRecallCurve => metrics.AreaUnderPrecisionRecallCurve, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs new file mode 100644 index 0000000000..2b7b6ac4d0 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettings.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class TrialSettings + { + public int TrialId { get; set; } + + public SweepableEstimatorPipeline Pipeline { get; set; } + + public string Schema { get; set; } + + public Parameter Parameter { get; set; } + + public AutoMLExperiment.AutoMLExperimentSettings ExperimentSettings { get; set; } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs new file mode 100644 index 0000000000..1f0a13835a --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class HyperParameterProposer : ITrialSettingsProposer + { + private readonly Dictionary _tuners; + + public HyperParameterProposer() + { + this._tuners = new Dictionary(); + } + + public TrialSettings Propose(TrialSettings settings) + { + if (!this._tuners.ContainsKey(settings.Schema)) + { + var t = settings.ExperimentSettings.TunerFactory(); + this._tuners.Add(settings.Schema, t); + } + + var tuner = this._tuners[settings.Schema]; + var parameter = tuner.Propose(settings.Pipeline.SearchSpace); + settings.Parameter = parameter; + + return settings; + } + + public void Update(TrialSettings settings, TrialResult result) + { + var schema = settings.Schema; + if (this._tuners.TryGetValue(schema, out var tuner)) + { + tuner.Update(settings.Parameter, result.Metric, settings.ExperimentSettings.EvaluateMetric.IsMaximize); + } + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs new file mode 100644 index 0000000000..ad73331fab --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/ITrialSettingsProposer.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML +{ + internal interface ITrialSettingsProposer + { + TrialSettings Propose(TrialSettings settings); + + void Update(TrialSettings parameter, TrialResult result); + } + + internal interface ISavableProposer : ITrialSettingsProposer + { + void SaveStatusToFile(string fileName); + + void LoadStatusFromFile(string fileName); + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs new file mode 100644 index 0000000000..76cdb491d4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -0,0 +1,269 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.ML.AutoML.CodeGen; +using Newtonsoft.Json; + +namespace Microsoft.ML.AutoML +{ + /// + /// propose sweepable estimator pipeline from a group of candidates using eci in flaml (https://arxiv.org/abs/1911.04706) + /// + internal class PipelineProposer : ISavableProposer + { + private readonly Dictionary _estimatorCost; + private Dictionary _learnerInitialCost; + + // total time spent on last best error for each learner. + private Dictionary _k1; + + // total time spent on second last best error for each learner. + private Dictionary _k2; + + // last best error for each learner. + private Dictionary _e1; + + // second last best error for each learner. + private Dictionary _e2; + + // flaml ECI + private Dictionary _eci; + private double _globalBestError; + + private readonly Random _rand; + private MultiModelPipeline _multiModelPipeline; + + public PipelineProposer(int seed) + { + this._estimatorCost = new Dictionary() + { + { EstimatorType.LightGbmRegression, 0.788 }, + { EstimatorType.FastTreeRegression, 0.382 }, + { EstimatorType.FastForestRegression, 0.374 }, + { EstimatorType.SdcaRegression, 0.566 }, + { EstimatorType.FastTreeTweedieRegression, 0.401 }, + { EstimatorType.LbfgsPoissonRegressionRegression, 4.73 }, + { EstimatorType.FastForestOva, 4.283 }, + { EstimatorType.FastTreeOva, 3.701 }, + { EstimatorType.LightGbmMulti, 14.765 }, + { EstimatorType.SdcaMaximumEntropyMulti, 1.129 }, + { EstimatorType.SdcaLogisticRegressionOva, 3.16 }, + { EstimatorType.LbfgsMaximumEntropyMulti, 7.980 }, + { EstimatorType.LbfgsLogisticRegressionOva, 11.513 }, + { EstimatorType.LightGbmBinary, 14.765 }, + { EstimatorType.FastTreeBinary, 3.701 }, + { EstimatorType.FastForestBinary, 4.283 }, + { EstimatorType.SdcaLogisticRegressionBinary, 3.16 }, + { EstimatorType.LbfgsLogisticRegressionBinary, 11.513 }, + { EstimatorType.ForecastBySsa, 1 }, + { EstimatorType.ImageClassificationMulti, 1 }, + { EstimatorType.MatrixFactorization, 1 }, + }; + this._rand = new Random(seed); + + // TODO + // use MultiModelPipeline from training configuration if possible. + this._multiModelPipeline = null; + } + + public TrialSettings Propose(TrialSettings settings) + { + this._multiModelPipeline = settings.ExperimentSettings.Pipeline; + this._learnerInitialCost = _multiModelPipeline.PipelineIds.ToDictionary(kv => kv, kv => this.GetEstimatedCostForPipeline(kv, this._multiModelPipeline)); + var pipelineIds = this._multiModelPipeline.PipelineIds; + + if (this._eci == null) + { + // initialize eci with the estimated cost and always start from pipeline which has lowest cost. + this._eci = pipelineIds.ToDictionary(kv => kv, kv => this.GetEstimatedCostForPipeline(kv, this._multiModelPipeline)); + settings.Schema = this._eci.OrderBy(kv => kv.Value).First().Key; + } + else + { + var probabilities = pipelineIds.Select(id => this._eci[id]).ToArray(); + probabilities = ArrayMath.Inverse(probabilities); + probabilities = ArrayMath.Normalize(probabilities); + + // sample + var randdouble = this._rand.NextDouble(); + var sum = 0.0; + // selected pipeline id index + int i; + + for (i = 0; i != pipelineIds.Length; ++i) + { + sum += ((double[])probabilities)[i]; + if (sum > randdouble) + { + break; + } + } + + settings.Schema = pipelineIds[i]; + } + + settings.Pipeline = this._multiModelPipeline.BuildSweepableEstimatorPipeline(settings.Schema); + return settings; + } + + public void SaveStatusToFile(string fileName) + { + using (var writer = new FileStream(fileName, FileMode.Create)) + { + this.SaveStatusToStream(writer); + } + } + + public void SaveStatusToStream(Stream stream) + { + var status = new Status() + { + K1 = this._k1, + K2 = this._k2, + E1 = this._e1, + E2 = this._e2, + Eci = this._eci, + GlobalBestError = this._globalBestError, + }; + + using (var fileWriter = new StreamWriter(stream)) + { + var json = JsonConvert.SerializeObject(status); + fileWriter.Write(json); + } + } + + public void LoadStatusFromFile(string fileName) + { + if (File.Exists(fileName)) + { + var json = File.ReadAllText(fileName); + var status = JsonConvert.DeserializeObject(json); + this._k1 = status.K1; + this._k2 = status.K2; + this._e1 = status.E1; + this._e2 = status.E2; + this._eci = status.Eci; + this._globalBestError = status.GlobalBestError; + } + } + + public void Update(TrialSettings parameter, TrialResult result) + { + var schema = parameter.Schema; + var error = this.CaculateError(result.Metric, result.TrialSettings.ExperimentSettings.EvaluateMetric.IsMaximize); + var duration = result.DurationInMilliseconds / 1000; + var pipelineIds = this._multiModelPipeline.PipelineIds; + var isSuccess = duration != 0; + + // if k1 is null, it means this is the first completed trial. + // in that case, initialize k1, k2, e1, e2 in the following way: + // k1: for every learner, k1[l] = c * duration where c is a ratio defined in learnerInitialCost + // k2: k2 = k1, which indicates the hypothesis that it costs the same time for learners to reach the next break through. + // e1: current error + // e2: 1.001*e1 + + if (isSuccess) + { + if (this._k1 == null) + { + this._k1 = pipelineIds.ToDictionary(id => id, id => duration * this._learnerInitialCost[id] / this._learnerInitialCost[schema]); + this._k2 = this._k1.ToDictionary(kv => kv.Key, kv => kv.Value); + this._e1 = pipelineIds.ToDictionary(id => id, id => error); + this._e2 = pipelineIds.ToDictionary(id => id, id => 1.05 * error); + this._globalBestError = error; + } + else if (error >= this._e1[schema]) + { + // if error is larger than current best error, which means there's no improvements for + // the last trial with the current learner. + // In that case, simply increase the total spent time since the last best error for that learner. + this._k1[schema] += duration; + } + else + { + // there's an improvement. + // k2 <= k1 && e2 <= e1, and update k1, e2. + this._k2[schema] = this._k1[schema]; + this._k1[schema] = duration; + this._e2[schema] = this._e1[schema]; + this._e1[schema] = error; + + // update global best error as well + if (error < this._globalBestError) + { + this._globalBestError = error; + } + } + + // update eci + var eci1 = Math.Max(this._k1[schema], this._k2[schema]); + var estimatorCostForBreakThrough = 2 * (error - this._globalBestError) / ((this._e2[schema] - this._e1[schema]) / (this._k2[schema] + this._k1[schema])); + this._eci[schema] = Math.Max(eci1, estimatorCostForBreakThrough); + } + else + { + // double eci of current trial twice of maxium ecis. + this._eci[schema] = this._eci.Select(kv => kv.Value).Max() * 2; + } + + // normalize eci + var sum = this._eci.Select(x => x.Value).Sum(); + this._eci = this._eci.Select(x => (x.Key, x.Value / sum)).ToDictionary(x => x.Key, x => x.Item2); + + // TODO + // save k1,k2,e1,e2,eci,bestError to training configuration + return; + } + + private double CaculateError(double loss, bool isMaximize) + { + return isMaximize ? 1 - loss : loss; + } + + private double GetEstimatedCostForPipeline(string kv, MultiModelPipeline multiModelPipeline) + { + var entity = Entity.FromExpression(kv); + + var estimatorTypes = entity.ValueEntities().Where(v => v is StringEntity s && s.Value != "Nil") + .Select(v => + { + var s = v as StringEntity; + var estimator = multiModelPipeline.Estimators[s.Value]; + return estimator.EstimatorType; + }); + + var res = 1; + + foreach (var estimatorType in estimatorTypes) + { + if (this._estimatorCost.ContainsKey(estimatorType)) + { + return this._estimatorCost[estimatorType]; + } + } + + return res; + } + + public class Status + { + public Dictionary K1 { get; set; } + + public Dictionary K2 { get; set; } + + public Dictionary E1 { get; set; } + + public Dictionary E2 { get; set; } + + public Dictionary Eci { get; set; } + + public double GlobalBestError { get; set; } + } + } +} diff --git a/src/Microsoft.ML.AutoML/Utils/ArrayMath.cs b/src/Microsoft.ML.AutoML/Utils/ArrayMath.cs new file mode 100644 index 0000000000..bb881a4cf4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Utils/ArrayMath.cs @@ -0,0 +1,168 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; + +namespace Microsoft.ML.AutoML +{ + public class ArrayMath + { + /* x + y */ + public static double[] Add(double[] xArray, double y) + { + return xArray.Select(x => x + y).ToArray(); + } + + /* np.argmax */ + public static int ArgMax(double[] array) + { + int index = 0; + for (int i = 1; i < array.Length; i++) + { + if (array[i] > array[index]) + { + index = i; + } + } + + return index; + } + + /* np.argsort */ + public static int[] ArgSort(double[] array) + { + return Enumerable.Range(0, array.Length).OrderBy(index => array[index]).ToArray(); + } + + /* np.clip */ + public static double[] Clip(double[] xArray, double min, double max) + { + return xArray.Select(x => Math.Min(Math.Max(x, min), max)).ToArray(); + } + + /* x / y */ + public static double[] Div(double[] xArray, double y) + { + return xArray.Select(x => x / y).ToArray(); + } + + /* x[y] */ + public static double[] Index(double[] array, int[] indices) + { + return indices.Select(index => array[index]).ToArray(); + } + + /* List.Insert */ + public static double[] Insert(double[] array, int index, double item) + { + double[] ret = new double[array.Length + 1]; + Array.Copy(array, 0, ret, 0, index); + ret[index] = item; + Array.Copy(array, index, ret, index + 1, array.Length - index); + return ret; + } + + /* np.linalg.norm */ + public static double Norm(double[] array) + { + double s = 0; + foreach (double x in array) + { + s += x * x; + } + + return Math.Sqrt(s); + } + + /* x * y */ + public static double[] Mul(double[] xArray, double y) + { + return xArray.Select(x => x * y).ToArray(); + } + + /* np.log */ + public static double[] Log(double[] xArray) + { + return xArray.Select(x => Math.Log(x)).ToArray(); + } + + /* x * y */ + public static double[] Mul(double[] xArray, double[] yArray) + { + return Enumerable.Zip(xArray, yArray, (x, y) => x * y).ToArray(); + } + + /* np.searchsorted */ + public static int SearchSorted(double[] array, double item) + { + int index = Array.BinarySearch(array, item); + return index >= 0 ? index : ~index; + } + + /* x - y */ + public static double[] Add(double[] xArray, double[] yArray) + { + return Enumerable.Zip(xArray, yArray, (x, y) => x + y).ToArray(); + } + + public static double[] Sub(double[] xArray, double[] yArray) + { + return Add(xArray, Mul(yArray, -1)); + } + + public static double[] Inverse(double[] array) + { + return array.Select(v => 1 / v).ToArray(); + } + + public static double[] Normalize(double[] array) + { + var sum = array.Sum(); + return array.Select(v => v / sum).ToArray(); + } + + public static double Rmse(double[] truth, double[] pred) + { + if (truth.Length != pred.Length) + { + throw new ArgumentException($"length doesn't match, {truth.Length} != {pred.Length}"); + } + + var diff = Enumerable.Range(0, truth.Length).Select(i => truth[i] - pred[i]).ToArray(); + var sqaure = diff.Select(x => x * x); + var mean = sqaure.Average(); + var rmse = Math.Sqrt(mean); + + return rmse; + } + + public static double Mape(double[] truth, double[] pred) + { + if (truth.Length != pred.Length) + { + throw new ArgumentException($"length doesn't match, {truth.Length} != {pred.Length}"); + } + + var diff = Enumerable.Range(0, truth.Length).Select(i => truth[i] - pred[i]).ToArray(); + var ape = diff.Select((x, i) => Math.Abs(x) / truth[i]); + var mape = ape.Average(); + + return mape; + } + + public static double Mae(double[] truth, double[] pred) + { + if (truth.Length != pred.Length) + { + throw new ArgumentException($"length doesn't match, {truth.Length} != {pred.Length}"); + } + + var diff = Enumerable.Range(0, truth.Length).Select(i => Math.Abs(truth[i] - pred[i])).ToArray(); + var mae = diff.Average(); + + return mae; + } + } +} diff --git a/src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs b/src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs new file mode 100644 index 0000000000..b0221b53c9 --- /dev/null +++ b/src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.SearchSpace +{ + internal interface ITuner + { + Parameter Propose(SearchSpace searchSpace); + + void Update(Parameter param, double metric, bool isMaximize); + } +} From 4f5567e5776bb763fddd6945234245e44bce8e86 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 21:57:48 -0700 Subject: [PATCH 07/25] fix bugs --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 38 ++++++++++--------- .../AutoMLExperiment/AutoMLExperiment.cs | 23 ++++++++++- .../AutoMLExperiment/IMonitor.cs | 2 +- .../Tuner/RandomTuner.cs | 19 ++++------ 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index 3b822b647f..dd825ac636 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -294,6 +294,11 @@ internal SweepableEstimator CreateSweepableEstimator(Func factory(context, param.AsType()), ss); } + internal AutoMLExperiment CreateExperiment() + { + return new AutoMLExperiment(this._context, new AutoMLExperiment.AutoMLExperimentSettings()); + } + internal SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) @@ -306,7 +311,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa fastTreeOption.LabelColumnName = labelColumnName; fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace(fastTreeOption))); } if (useFastForest) @@ -315,7 +320,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa fastForestOption.LabelColumnName = labelColumnName; fastForestOption.FeatureColumnName = featureColumnName; fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace(fastForestOption))); } if (useLgbm) @@ -324,7 +329,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa lgbmOption.LabelColumnName = labelColumnName; lgbmOption.FeatureColumnName = featureColumnName; lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace(lgbmOption))); } if (useLbfgs) @@ -333,7 +338,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa lbfgsOption.LabelColumnName = labelColumnName; lbfgsOption.FeatureColumnName = featureColumnName; lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); } if (useSdca) @@ -342,7 +347,7 @@ internal SweepableEstimator[] BinaryClassification(string labelColumnName = Defa sdcaOption.LabelColumnName = labelColumnName; sdcaOption.FeatureColumnName = featureColumnName; sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } return res.ToArray(); @@ -360,7 +365,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau fastTreeOption.LabelColumnName = labelColumnName; fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace(fastTreeOption))); } if (useFastForest) @@ -369,7 +374,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau fastForestOption.LabelColumnName = labelColumnName; fastForestOption.FeatureColumnName = featureColumnName; fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace(fastForestOption))); } if (useLgbm) @@ -378,7 +383,7 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau lgbmOption.LabelColumnName = labelColumnName; lgbmOption.FeatureColumnName = featureColumnName; lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace(lgbmOption))); } if (useLbfgs) @@ -387,8 +392,8 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau lbfgsOption.LabelColumnName = labelColumnName; lbfgsOption.FeatureColumnName = featureColumnName; lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); - res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); + res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); } if (useSdca) @@ -397,8 +402,8 @@ internal SweepableEstimator[] MultiClassification(string labelColumnName = Defau sdcaOption.LabelColumnName = labelColumnName; sdcaOption.FeatureColumnName = featureColumnName; sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); - res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } return res.ToArray(); @@ -417,7 +422,6 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); - res.Add(SweepableEstimatorFactory.CreateFastTreeTweedieRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); } if (useFastForest) @@ -426,7 +430,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN fastForestOption.LabelColumnName = labelColumnName; fastForestOption.FeatureColumnName = featureColumnName; fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace(fastForestOption))); } if (useLgbm) @@ -435,7 +439,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN lgbmOption.LabelColumnName = labelColumnName; lgbmOption.FeatureColumnName = featureColumnName; lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace(lgbmOption))); } if (useLbfgs) @@ -444,7 +448,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN lbfgsOption.LabelColumnName = labelColumnName; lbfgsOption.FeatureColumnName = featureColumnName; lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace(lbfgsOption))); } if (useSdca) @@ -453,7 +457,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN sdcaOption.LabelColumnName = labelColumnName; sdcaOption.FeatureColumnName = featureColumnName; sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace(sdcaOption))); } return res.ToArray(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index c519f76360..d446fa60f1 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -61,6 +62,24 @@ public AutoMLExperiment SetTunerFactory(Func tunerFactory) return this; } + public AutoMLExperiment SetMonitor(IMonitor monitor) + { + this._settings.Monitor = monitor; + return this; + } + + public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) + { + this._settings.Pipeline = pipeline; + return this; + } + + public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) + { + this._settings.Pipeline = new MultiModelPipeline().Append(pipeline.Estimators.ToArray()); + return this; + } + public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") { this._settings.EvaluateMetric = new BinaryMetricSettings() @@ -107,8 +126,10 @@ public Task Run() var cts = new CancellationTokenSource(); cts.CancelAfter((int)this._settings.MaxExperimentTimeInSeconds * 1000); this._settings.CancellationToken.Register(() => cts.Cancel()); + var ct = cts.Token; + ct.Register(() => this._context.CancelExecution()); - return this.RunAsync(cts.Token); + return this.RunAsync(ct); } private async Task RunAsync(CancellationToken ct) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 3ac894ac78..36a2d38a6b 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -41,7 +41,7 @@ public void ReportBestTrial(TrialResult result) public void ReportCompletedTrial(TrialResult result) { - this._logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline}"); + this._logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {result.TrialSettings.Pipeline} - Duration: {result.DurationInMilliseconds}"); this._completedTrials.Add(result); } diff --git a/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs b/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs index 7c22e735df..dbb5cd16ae 100644 --- a/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs +++ b/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs @@ -7,28 +7,25 @@ namespace Microsoft.ML.SearchSpace.Tuner { - internal sealed class RandomTuner + internal sealed class RandomTuner : ITuner { - private readonly SearchSpace _searchSpace; private readonly Random _rnd; - public RandomTuner(SearchSpace searchSpace) + public RandomTuner() { - this._searchSpace = searchSpace; this._rnd = new Random(); } - public RandomTuner(SearchSpace searchSpace, int seed) + public Parameter Propose(SearchSpace searchSpace) { - this._searchSpace = searchSpace; - this._rnd = new Random(seed); + var d = searchSpace.FeatureSpaceDim; + var featureVec = Enumerable.Repeat(0, d).Select(i => this._rnd.NextDouble()).ToArray(); + return searchSpace.SampleFromFeatureSpace(featureVec); } - public Parameter Propose() + public void Update(Parameter param, double metric, bool isMaximize) { - var d = this._searchSpace.FeatureSpaceDim; - var featureVec = Enumerable.Repeat(0, d).Select(i => this._rnd.NextDouble()).ToArray(); - return this._searchSpace.SampleFromFeatureSpace(featureVec); + // do nothing } } } From c05794686982819b1d6c54bf1651a6e0d643e320 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 16 Mar 2022 10:18:57 -0700 Subject: [PATCH 08/25] add append --- src/Microsoft.ML.AutoML/API/SweepableExtension.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index 1848d8c128..20193642ba 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -39,6 +39,14 @@ public static MultiModelPipeline Append(this IEstimator estimator, return multiModelPipeline; } + public static MultiModelPipeline Append(this MultiModelPipeline pipeline, IEstimator estimator) + { + var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); + var multiModelPipeline = pipeline.Append(sweepableEstimator); + + return multiModelPipeline; + } + public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) { var multiModelPipeline = new MultiModelPipeline(); From 3336402c8bfd0fe1725ea4e2c569e014758547eb Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 16 Mar 2022 10:19:11 -0700 Subject: [PATCH 09/25] add titanic example --- .../Microsoft.ML.AutoML.Samples.csproj | 1 + docs/samples/Microsoft.ML.AutoML.Samples/Program.cs | 2 ++ src/Microsoft.ML.AutoML/Assembly.cs | 2 +- src/Microsoft.ML.SearchSpace/Assembly.cs | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj index 9d511df6a5..1c4da82682 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj @@ -10,6 +10,7 @@ + diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs index 4342873c64..6ecc9663c9 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs @@ -8,6 +8,8 @@ public static void Main(string[] args) { try { + TitanicExperiment.Run(); + RecommendationExperiment.Run(); Console.Clear(); diff --git a/src/Microsoft.ML.AutoML/Assembly.cs b/src/Microsoft.ML.AutoML/Assembly.cs index c669817f47..3d1496cead 100644 --- a/src/Microsoft.ML.AutoML/Assembly.cs +++ b/src/Microsoft.ML.AutoML/Assembly.cs @@ -14,4 +14,4 @@ [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService.Gpu, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] - +[assembly: InternalsVisibleTo("Microsoft.ML.AutoML.Samples, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] diff --git a/src/Microsoft.ML.SearchSpace/Assembly.cs b/src/Microsoft.ML.SearchSpace/Assembly.cs index 39b50da899..db60de65f1 100644 --- a/src/Microsoft.ML.SearchSpace/Assembly.cs +++ b/src/Microsoft.ML.SearchSpace/Assembly.cs @@ -7,4 +7,5 @@ [assembly: InternalsVisibleTo("Microsoft.ML.AutoML.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] [assembly: InternalsVisibleTo("Microsoft.ML.SearchSpace.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] [assembly: InternalsVisibleTo("Microsoft.ML.AutoML, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] +[assembly: InternalsVisibleTo("Microsoft.ML.AutoML.Samples, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] From 0fe559e1f355a618846e6ca0e59780721c2f9f81 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 17:12:05 -0700 Subject: [PATCH 10/25] use di to inject services --- .../AutoMLExperiment/AutoMLExperiment.cs | 60 ++++++++++++++----- .../AutoMLExperiment/IMonitor.cs | 4 +- .../AutoMLExperiment/TrialRunnerFactory.cs | 39 ++++++++++++ .../Microsoft.ML.AutoML.csproj | 1 + 4 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index d446fa60f1..f8acc7d807 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -8,6 +8,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.SearchSpace; @@ -20,11 +22,23 @@ internal class AutoMLExperiment private readonly MLContext _context; private double _bestError = double.MaxValue; private TrialResult _bestTrialResult = null; + private IServiceCollection _serviceCollection; public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { this._context = context; this._settings = settings; + this.InitializeServiceCollection(); + } + + private void InitializeServiceCollection() + { + this._serviceCollection = new ServiceCollection(); + this._serviceCollection.TryAddSingleton(this._context); + this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); } public AutoMLExperiment SetTrainingTimeInSeconds(int trainingTimeInSeconds) @@ -64,7 +78,16 @@ public AutoMLExperiment SetTunerFactory(Func tunerFactory) public AutoMLExperiment SetMonitor(IMonitor monitor) { - this._settings.Monitor = monitor; + var descriptor = new ServiceDescriptor(typeof(IMonitor), monitor); + if (this._serviceCollection.Contains(descriptor)) + { + this._serviceCollection.Replace(descriptor); + } + else + { + this._serviceCollection.Add(descriptor); + } + return this; } @@ -74,6 +97,21 @@ public AutoMLExperiment SetPipeline(MultiModelPipeline pipeline) return this; } + public AutoMLExperiment SetTrialRunnerFactory(ITrialRunnerFactory factory) + { + var descriptor = new ServiceDescriptor(typeof(ITrialRunnerFactory), factory); + if (this._serviceCollection.Contains(descriptor)) + { + this._serviceCollection.Replace(descriptor); + } + else + { + this._serviceCollection.Add(descriptor); + } + + return this; + } + public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) { this._settings.Pipeline = new MultiModelPipeline().Append(pipeline.Estimators.ToArray()); @@ -134,9 +172,12 @@ public Task Run() private async Task RunAsync(CancellationToken ct) { + var serviceProvider = this._serviceCollection.BuildServiceProvider(); + var monitor = serviceProvider.GetService(); var trialNum = 0; var pipelineProposer = new PipelineProposer(this._settings.Seed ?? 0); var hyperParameterProposer = new HyperParameterProposer(); + var runnerFactory = serviceProvider.GetService(); while (true) { @@ -153,17 +194,10 @@ private async Task RunAsync(CancellationToken ct) setting = pipelineProposer.Propose(setting); setting = hyperParameterProposer.Propose(setting); - - ITrialRunner runner = (this._settings.DatasetSettings, this._settings.EvaluateMetric) switch - { - (CrossValidateDatasetSettings, BinaryMetricSettings) => new BinaryClassificationCVRunner(), - (TrainTestDatasetSettings, BinaryMetricSettings) => new BinaryClassificationTrainTestRunner(), - _ => throw new NotImplementedException(), - }; - - this._settings.Monitor.ReportRunningTrial(setting); + monitor.ReportRunningTrial(setting); + var runner = runnerFactory.CreateTrialRunner(setting); var trialResult = runner.Run(this._context, setting); - this._settings.Monitor.ReportCompletedTrial(trialResult); + monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); pipelineProposer.Update(setting, trialResult); @@ -172,7 +206,7 @@ private async Task RunAsync(CancellationToken ct) { this._bestTrialResult = trialResult; this._bestError = error; - this._settings.Monitor.ReportBestTrial(trialResult); + monitor.ReportBestTrial(trialResult); } } @@ -205,8 +239,6 @@ public class AutoMLExperimentSettings : ExperimentSettings public Func TunerFactory { get; set; } - public IMonitor Monitor { get; set; } - public int? Seed { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index 36a2d38a6b..5d25f7645f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -24,12 +24,14 @@ internal interface IMonitor internal class MLContextMonitor : IMonitor { private readonly MLContext _context; + private readonly IServiceProvider _serviceProvider; private readonly IChannel _logger; private readonly List _completedTrials; - public MLContextMonitor(MLContext context) + public MLContextMonitor(MLContext context, IServiceProvider provider) { this._context = context; + this._serviceProvider = provider; this._logger = ((IChannelProvider)context).Start(nameof(AutoMLExperiment)); this._completedTrials = new List(); } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs new file mode 100644 index 0000000000..38d954e8fa --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.DependencyInjection; + +#nullable enable +namespace Microsoft.ML.AutoML +{ + internal interface ITrialRunnerFactory + { + ITrialRunner? CreateTrialRunner(TrialSettings settings); + } + + internal class TrialRunnerFactory : ITrialRunnerFactory + { + private readonly IServiceProvider _provider; + + public TrialRunnerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITrialRunner? CreateTrialRunner(TrialSettings settings) + { + ITrialRunner? runner = (settings.ExperimentSettings.DatasetSettings, settings.ExperimentSettings.EvaluateMetric) switch + { + (CrossValidateDatasetSettings, BinaryMetricSettings) => this._provider.GetService(), + (TrainTestDatasetSettings, BinaryMetricSettings) => this._provider.GetService(), + _ => throw new NotImplementedException(), + }; + + return runner; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 6d79838dff..e09b1ed3a7 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -20,6 +20,7 @@ true + From 9a484114fd7f9052e888651e596fea01b5d5f349 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 23 Mar 2022 16:12:54 -0700 Subject: [PATCH 11/25] add cfo, random and grid search tuners and tests --- .../AutoMLExperiment/AutoMLExperiment.cs | 31 ++- .../HyperParameterProposer.cs | 13 +- .../TrialSettingsProposer/PipelineProposer.cs | 5 +- .../AutoMLExperiment/TunerFactory.cs | 71 +++++ src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs | 122 +++++++++ src/Microsoft.ML.AutoML/Tuner/Flow2.cs | 159 +++++++++++ .../Tuner/GridSearchTuner.cs | 42 +++ .../Tuner/ITuner.cs | 10 +- .../Tuner/RandomSearchTuner.cs | 33 +++ src/Microsoft.ML.AutoML/Tuner/SearchThread.cs | 71 +++++ .../Utils/RandomNumberGenerator.cs | 79 ++++++ .../Tuner/RandomTuner.cs | 2 +- test/Microsoft.ML.AutoML.Tests/TunerTests.cs | 249 ++++++++++++++++++ 13 files changed, 864 insertions(+), 23 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs create mode 100644 src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs create mode 100644 src/Microsoft.ML.AutoML/Tuner/Flow2.cs create mode 100644 src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs rename src/{Microsoft.ML.SearchSpace => Microsoft.ML.AutoML}/Tuner/ITuner.cs (53%) create mode 100644 src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs create mode 100644 src/Microsoft.ML.AutoML/Tuner/SearchThread.cs create mode 100644 src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs create mode 100644 test/Microsoft.ML.AutoML.Tests/TunerTests.cs diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index f8acc7d807..bea18e723e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -13,6 +13,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.SearchSpace; +using Microsoft.ML.SearchSpace.Tuner; namespace Microsoft.ML.AutoML { @@ -22,23 +23,26 @@ internal class AutoMLExperiment private readonly MLContext _context; private double _bestError = double.MaxValue; private TrialResult _bestTrialResult = null; - private IServiceCollection _serviceCollection; + private readonly IServiceCollection _serviceCollection; public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { this._context = context; this._settings = settings; - this.InitializeServiceCollection(); + this._serviceCollection = new ServiceCollection(); } private void InitializeServiceCollection() { - this._serviceCollection = new ServiceCollection(); this._serviceCollection.TryAddSingleton(this._context); + this._serviceCollection.TryAddSingleton(this._settings); this._serviceCollection.TryAddSingleton(); this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddSingleton(); this._serviceCollection.TryAddTransient(); this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddScoped(); + this._serviceCollection.TryAddScoped(); } public AutoMLExperiment SetTrainingTimeInSeconds(int trainingTimeInSeconds) @@ -69,9 +73,18 @@ public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) return this; } - public AutoMLExperiment SetTunerFactory(Func tunerFactory) + public AutoMLExperiment SetTunerFactory() + where TTunerFactory : ITunerFactory { - this._settings.TunerFactory = tunerFactory; + var descriptor = new ServiceDescriptor(typeof(ITunerFactory), typeof(TTunerFactory), ServiceLifetime.Singleton); + if (this._serviceCollection.Contains(descriptor)) + { + this._serviceCollection.Replace(descriptor); + } + else + { + this._serviceCollection.Add(descriptor); + } return this; } @@ -172,11 +185,12 @@ public Task Run() private async Task RunAsync(CancellationToken ct) { + this.InitializeServiceCollection(); var serviceProvider = this._serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); var trialNum = 0; - var pipelineProposer = new PipelineProposer(this._settings.Seed ?? 0); - var hyperParameterProposer = new HyperParameterProposer(); + var pipelineProposer = serviceProvider.GetService(); + var hyperParameterProposer = serviceProvider.GetService(); var runnerFactory = serviceProvider.GetService(); while (true) @@ -224,7 +238,6 @@ private void ValidateSettings() { Contracts.Assert(this._settings.MaxExperimentTimeInSeconds > 0, $"{nameof(ExperimentSettings.MaxExperimentTimeInSeconds)} must be larger than 0"); Contracts.Assert(this._settings.DatasetSettings != null, $"{nameof(this._settings.DatasetSettings)} must be not null"); - Contracts.Assert(this._settings.TunerFactory != null, $"{nameof(this._settings.TunerFactory)} must be not null"); Contracts.Assert(this._settings.EvaluateMetric != null, $"{nameof(this._settings.EvaluateMetric)} must be not null"); } @@ -237,8 +250,6 @@ public class AutoMLExperimentSettings : ExperimentSettings public MultiModelPipeline Pipeline { get; set; } - public Func TunerFactory { get; set; } - public int? Seed { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs index 1f0a13835a..3e98895f54 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/HyperParameterProposer.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.SearchSpace; namespace Microsoft.ML.AutoML @@ -10,22 +12,25 @@ namespace Microsoft.ML.AutoML internal class HyperParameterProposer : ITrialSettingsProposer { private readonly Dictionary _tuners; + private readonly IServiceProvider _provider; - public HyperParameterProposer() + public HyperParameterProposer(IServiceProvider provider) { this._tuners = new Dictionary(); + this._provider = provider; } public TrialSettings Propose(TrialSettings settings) { + var tunerFactory = this._provider.GetService(); if (!this._tuners.ContainsKey(settings.Schema)) { - var t = settings.ExperimentSettings.TunerFactory(); + var t = tunerFactory.CreateTuner(settings); this._tuners.Add(settings.Schema, t); } var tuner = this._tuners[settings.Schema]; - var parameter = tuner.Propose(settings.Pipeline.SearchSpace); + var parameter = tuner.Propose(settings); settings.Parameter = parameter; return settings; @@ -36,7 +41,7 @@ public void Update(TrialSettings settings, TrialResult result) var schema = settings.Schema; if (this._tuners.TryGetValue(schema, out var tuner)) { - tuner.Update(settings.Parameter, result.Metric, settings.ExperimentSettings.EvaluateMetric.IsMaximize); + tuner.Update(result); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index 76cdb491d4..d0b81f39a2 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -8,6 +8,7 @@ using System.Linq; using Microsoft.ML.AutoML.CodeGen; using Newtonsoft.Json; +using static Microsoft.ML.AutoML.AutoMLExperiment; namespace Microsoft.ML.AutoML { @@ -38,7 +39,7 @@ internal class PipelineProposer : ISavableProposer private readonly Random _rand; private MultiModelPipeline _multiModelPipeline; - public PipelineProposer(int seed) + public PipelineProposer(AutoMLExperimentSettings settings) { this._estimatorCost = new Dictionary() { @@ -64,7 +65,7 @@ public PipelineProposer(int seed) { EstimatorType.ImageClassificationMulti, 1 }, { EstimatorType.MatrixFactorization, 1 }, }; - this._rand = new Random(seed); + this._rand = new Random(settings.Seed ?? 0); // TODO // use MultiModelPipeline from training configuration if possible. diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs new file mode 100644 index 0000000000..e3823fcbc9 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.SearchSpace.Tuner; + +namespace Microsoft.ML.AutoML +{ + internal interface ITunerFactory + { + ITuner CreateTuner(TrialSettings settings); + } + + internal class CfoTunerFactory : ITunerFactory + { + private readonly IServiceProvider _provider; + + public CfoTunerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITuner CreateTuner(TrialSettings settings) + { + var experimentSetting = this._provider.GetService(); + var searchSpace = settings.Pipeline.SearchSpace; + var initParameter = settings.Pipeline.Parameter; + var isMaximize = experimentSetting.EvaluateMetric.IsMaximize; + + return new CfoTuner(searchSpace, initParameter, !isMaximize); + } + } + + internal class RandomTunerFactory : ITunerFactory + { + private readonly IServiceProvider _provider; + + public RandomTunerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITuner CreateTuner(TrialSettings settings) + { + var searchSpace = settings.Pipeline.SearchSpace; + + return new RandomSearchTuner(searchSpace); + } + } + + internal class GridSearchTunerFactory : ITunerFactory + { + private readonly IServiceProvider _provider; + + public GridSearchTunerFactory(IServiceProvider provider) + { + this._provider = provider; + } + + public ITuner CreateTuner(TrialSettings settings) + { + var searchSpace = settings.Pipeline.SearchSpace; + + return new GridSearchTuner(searchSpace); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs b/src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs new file mode 100644 index 0000000000..2cf6ea7a43 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs @@ -0,0 +1,122 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.ML.AutoMLService; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class CfoTuner : ITuner + { + private readonly RandomNumberGenerator _rng = new RandomNumberGenerator(); + private readonly SearchSpace.SearchSpace _searchSpace; + private readonly bool _minimize; + private readonly Flow2 _localSearch; + private readonly Dictionary _searchThreadPool = new Dictionary(); + private int _currentThreadId; + private readonly Dictionary _trialProposedBy = new Dictionary(); + + private readonly Dictionary _configs = new Dictionary(); + private readonly double[] _lsBoundMax; + private readonly double[] _lsBoundMin; + private bool _initUsed = false; + private double _bestMetric; + + public CfoTuner(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true) + { + this._searchSpace = searchSpace; + this._minimize = minimizeMode; + + this._localSearch = new Flow2(searchSpace, initValue, true); + this._currentThreadId = 0; + this._lsBoundMin = this._searchSpace.MappingToFeatureSpace(initValue); + this._lsBoundMax = this._searchSpace.MappingToFeatureSpace(initValue); + this._initUsed = false; + this._bestMetric = double.MaxValue; + } + + public Parameter Propose(TrialSettings settings) + { + var trialId = settings.TrialId; + if (this._initUsed) + { + var searchThread = this._searchThreadPool[this._currentThreadId]; + this._configs[trialId] = this._searchSpace.MappingToFeatureSpace(searchThread.Suggest(trialId)); + this._trialProposedBy[trialId] = this._currentThreadId; + } + else + { + this._configs[trialId] = this.CreateInitConfigFromAdmissibleRegion(); + this._trialProposedBy[trialId] = this._currentThreadId; + } + + var param = this._configs[trialId]; + return this._searchSpace.SampleFromFeatureSpace(param); + } + + public Parameter BestConfig { get; set; } + + public void Update(TrialResult result) + { + var trialId = result.TrialSettings.TrialId; + var metric = result.Metric; + metric = this._minimize ? metric : -metric; + if (metric < this._bestMetric) + { + this.BestConfig = this._searchSpace.SampleFromFeatureSpace(this._configs[trialId]); + this._bestMetric = metric; + } + + var cost = result.DurationInMilliseconds; + int threadId = this._trialProposedBy[trialId]; + if (this._searchThreadPool.Count == 0) + { + var initParameter = this._searchSpace.SampleFromFeatureSpace(this._configs[trialId]); + this._searchThreadPool[this._currentThreadId] = this._localSearch.CreateSearchThread(initParameter, metric, cost); + this._initUsed = true; + this.UpdateAdmissibleRegion(this._configs[trialId]); + } + else + { + this._searchThreadPool[threadId].OnTrialComplete(trialId, metric, cost); + if (this._searchThreadPool[threadId].IsConverged) + { + this._searchThreadPool.Remove(threadId); + this._currentThreadId += 1; + this._initUsed = false; + } + } + } + + private void UpdateAdmissibleRegion(double[] config) + { + for (int i = 0; i != config.Length; ++i) + { + if (config[i] < this._lsBoundMin[i]) + { + this._lsBoundMin[i] = config[i]; + continue; + } + + if (config[i] > this._lsBoundMax[i]) + { + this._lsBoundMax[i] = config[i]; + continue; + } + } + } + + private double[] CreateInitConfigFromAdmissibleRegion() + { + var res = new double[this._lsBoundMax.Length]; + for (int i = 0; i != this._lsBoundMax.Length; ++i) + { + res[i] = this._rng.Uniform(this._lsBoundMin[i], this._lsBoundMax[i]); + } + + return res; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/Flow2.cs b/src/Microsoft.ML.AutoML/Tuner/Flow2.cs new file mode 100644 index 0000000000..109d5d1cce --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/Flow2.cs @@ -0,0 +1,159 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Microsoft.ML.AutoMLService; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class Flow2 + { + private const double _stepSize = 0.1; + private const double _stepLowerBound = 0.0001; + + private readonly RandomNumberGenerator _rng = new RandomNumberGenerator(); + + public double? BestObj = null; + public double? CostIncumbent = null; + private readonly Parameter _initConfig; + private double _step; + + private readonly SearchSpace.SearchSpace _searchSpace; + private readonly bool _minimize; + private readonly double _convergeSpeed = 2; + private Parameter _bestConfig; + private readonly Dictionary _configs = new Dictionary(); + private double _costComplete4Incumbent = 0; + private readonly int _dim; + private double[] _directionTried = null; + private double[] _incumbent; + private int _numAllowed4Incumbent = 0; + private readonly Dictionary _proposedBy = new Dictionary(); + private readonly double _stepUpperBound; + private int _trialCount = 1; + + public Flow2(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true, double convergeSpeed = 1.5) + { + this._searchSpace = searchSpace; + this._minimize = minimizeMode; + + this._initConfig = initValue; + this._bestConfig = this._initConfig; + this._incumbent = this._searchSpace.MappingToFeatureSpace(this._bestConfig); + this._dim = this._searchSpace.Count; + this._numAllowed4Incumbent = 2 * this._dim; + this._step = _stepSize * Math.Sqrt(this._dim); + this._stepUpperBound = Math.Sqrt(this._dim); + this._convergeSpeed = convergeSpeed; + if (this._step > this._stepUpperBound) + { + this._step = this._stepUpperBound; + } + } + + public bool IsConverged + { + get => this._step < _stepLowerBound; + } + + public Parameter BestConfig + { + get => this._bestConfig; + } + + public SearchThread CreateSearchThread(Parameter config, double metric, double cost) + { + var flow2 = new Flow2(this._searchSpace, config, this._minimize, convergeSpeed: this._convergeSpeed); + flow2.BestObj = metric; + flow2.CostIncumbent = cost; + return new SearchThread(flow2); + } + + public Parameter Suggest(int trialId) + { + this._numAllowed4Incumbent -= 1; + double[] move; + if (this._directionTried != null) + { + move = ArrayMath.Sub(this._incumbent, this._directionTried); + + this._directionTried = null; + } + else + { + this._directionTried = this.RandVectorSphere(); + move = ArrayMath.Add(this._incumbent, this._directionTried); + } + + move = this.Project(move); + var config = this._searchSpace.SampleFromFeatureSpace(move); + this._proposedBy[trialId] = this._incumbent; + this._configs[trialId] = config; + return config; + } + + public void ReceiveTrialResult(int trialId, double metric, double cost) + { + this._trialCount += 1; + double obj = metric; // flipped in BlendSearch + //double obj = minimize ? metric : -metric; + if (this.BestObj == null || obj < this.BestObj) + { + this.BestObj = obj; + this._bestConfig = this._configs[trialId]; + this._incumbent = this._searchSpace.MappingToFeatureSpace(this._bestConfig); + this.CostIncumbent = cost; + this._costComplete4Incumbent = 0; + this._numAllowed4Incumbent = 2 * this._dim; + this._proposedBy.Clear(); + this._step *= this._convergeSpeed; + this._step = Math.Min(this._step, this._stepUpperBound); + this._directionTried = null; + return; + } + else + { + this._costComplete4Incumbent += cost; + if (this._numAllowed4Incumbent == 0) + { + this._numAllowed4Incumbent = 2; + if (!this.IsConverged) + { + this._step /= this._convergeSpeed; + } + } + } + } + + private double[] RandVectorSphere() + { + double[] vec = this._rng.Normal(0, 1, this._searchSpace.FeatureSpaceDim); + double mag = ArrayMath.Norm(vec); + vec = ArrayMath.Mul(vec, this._step / mag); + + return vec; + } + + private double[] Project(double[] move) + { + return move.Select(x => + { + if (x < 0) + { + x = 0; + } + else if (x > 1) + { + x = 0.99999999; + } + + return x; + }).ToArray(); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs new file mode 100644 index 0000000000..e22450144c --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/GridSearchTuner.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.SearchSpace.Tuner; + +namespace Microsoft.ML.AutoML +{ + internal class GridSearchTuner : ITuner + { + private readonly SearchSpace.Tuner.GridSearchTuner _tuner; + private IEnumerator _enumerator; + + public GridSearchTuner(SearchSpace.SearchSpace searchSpace) + { + this._tuner = new SearchSpace.Tuner.GridSearchTuner(searchSpace); + this._enumerator = this._tuner.Propose().GetEnumerator(); + } + public Parameter Propose(TrialSettings settings) + { + if (!this._enumerator.MoveNext()) + { + this._enumerator = this._tuner.Propose().GetEnumerator(); + return this.Propose(settings); + } + else + { + var res = this._enumerator.Current; + return res; + } + } + + public void Update(TrialResult result) + { + return; + } + } +} diff --git a/src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs b/src/Microsoft.ML.AutoML/Tuner/ITuner.cs similarity index 53% rename from src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs rename to src/Microsoft.ML.AutoML/Tuner/ITuner.cs index b0221b53c9..5844232c33 100644 --- a/src/Microsoft.ML.SearchSpace/Tuner/ITuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/ITuner.cs @@ -2,16 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.Collections.Generic; -using System.Text; +using Microsoft.ML.SearchSpace; -namespace Microsoft.ML.SearchSpace +namespace Microsoft.ML.AutoML { internal interface ITuner { - Parameter Propose(SearchSpace searchSpace); + Parameter Propose(TrialSettings settings); - void Update(Parameter param, double metric, bool isMaximize); + void Update(TrialResult result); } } diff --git a/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs new file mode 100644 index 0000000000..dfde0ade86 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/RandomSearchTuner.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.SearchSpace.Tuner; + +namespace Microsoft.ML.AutoML +{ + internal class RandomSearchTuner : ITuner + { + private readonly RandomTuner _tuner; + private readonly SearchSpace.SearchSpace _searchSpace; + + public RandomSearchTuner(SearchSpace.SearchSpace searchSpace) + { + this._tuner = new RandomTuner(); + this._searchSpace = searchSpace; + } + public Parameter Propose(TrialSettings settings) + { + return this._tuner.Propose(this._searchSpace); + } + + public void Update(TrialResult result) + { + return; + } + } +} diff --git a/src/Microsoft.ML.AutoML/Tuner/SearchThread.cs b/src/Microsoft.ML.AutoML/Tuner/SearchThread.cs new file mode 100644 index 0000000000..7889549ded --- /dev/null +++ b/src/Microsoft.ML.AutoML/Tuner/SearchThread.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.ML.SearchSpace; +using FlamlParameters = System.Collections.Generic.Dictionary; + +namespace Microsoft.ML.AutoML +{ + internal class SearchThread + { + private const double _eps = 1e-10; + + private readonly Flow2 _searchAlg; + + private double _costBest; + private double _costBest1; + private double _costBest2; + private double _costLast; + private double _costTotal; + private double _objBest1; + private double _objBest2; + private double _speed = 0; + + public SearchThread(Flow2 searchAlgorithm) + { + this._searchAlg = searchAlgorithm; + this._costLast = searchAlgorithm.CostIncumbent == null ? 0 : (double)searchAlgorithm.CostIncumbent; + this._costTotal = this._costLast; + this._costBest = this._costLast; + this._costBest1 = this._costLast; + this._costBest2 = 0; + this._objBest1 = searchAlgorithm.BestObj == null ? double.PositiveInfinity : (double)searchAlgorithm.BestObj; + this._objBest2 = this._objBest1; + } + + public bool IsConverged { get => this._searchAlg.IsConverged; } + + public Flow2 SearchArg { get => this._searchAlg; } + + public void OnTrialComplete(int trialId, double metric, double cost) + { + this._searchAlg.ReceiveTrialResult(trialId, metric, cost); + this._costLast = cost; + this._costTotal += cost; + if (metric < this._objBest1) + { + this._costBest2 = this._costBest1; + this._costBest1 = this._costTotal; + this._objBest2 = double.IsInfinity(this._objBest1) ? metric : this._objBest1; + this._objBest1 = metric; + this._costBest = this._costLast; + } + + if (this._objBest2 > this._objBest1) + { + this._speed = (this._objBest2 - this._objBest1) / (this._costTotal - this._costBest2 + _eps); + } + else + { + this._speed = 0; + } + } + + public Parameter Suggest(int trialId) + { + return this._searchAlg.Suggest(trialId); + } + } +} diff --git a/src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs b/src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs new file mode 100644 index 0000000000..1528432231 --- /dev/null +++ b/src/Microsoft.ML.AutoML/Utils/RandomNumberGenerator.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.ML.AutoMLService +{ + internal class RandomNumberGenerator + { + private readonly Random _random; + + public RandomNumberGenerator() + { + this._random = new Random(); + } + + public RandomNumberGenerator(int seed) + { + this._random = new Random(seed); + } + + public int Integer(int high) + { + return this._random.Next(high); + } + + public double Uniform(double low, double high) + { + return (this._random.NextDouble() * (high - low)) + low; + } + + public double Normal(double location, double scale) + { + double u = 1 - this.Uniform(0, 1); + double v = 1 - this.Uniform(0, 1); + double std = Math.Sqrt(-2.0 * Math.Log(u)) * Math.Sin(2.0 * Math.PI * v); + return location + (std * scale); + } + + public double[] Normal(double location, double scale, int size) + { + double[] ret = new double[size]; + for (int i = 0; i < size; i++) + { + ret[i] = this.Normal(location, scale); + } + + return ret; + } + + public int Categorical(double[] possibility) + { + double x = this.Uniform(0, 1); + for (int i = 0; i < possibility.Length; i++) + { + x -= possibility[i]; + if (x < 0) { return i; } + } + + return possibility.Length - 1; + } + + public int[] Categorical(double[] possibility, int size) + { + int[] ret = new int[size]; + for (int i = 0; i < ret.Length; i++) + { + ret[i] = this.Categorical(possibility); + } + + return ret; + } + } +} diff --git a/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs b/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs index dbb5cd16ae..51cbf35c60 100644 --- a/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs +++ b/src/Microsoft.ML.SearchSpace/Tuner/RandomTuner.cs @@ -7,7 +7,7 @@ namespace Microsoft.ML.SearchSpace.Tuner { - internal sealed class RandomTuner : ITuner + internal sealed class RandomTuner { private readonly Random _rnd; diff --git a/test/Microsoft.ML.AutoML.Tests/TunerTests.cs b/test/Microsoft.ML.AutoML.Tests/TunerTests.cs new file mode 100644 index 0000000000..4a01be897c --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/TunerTests.cs @@ -0,0 +1,249 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Threading; +using FluentAssertions; +using Microsoft.ML.SearchSpace; +using Microsoft.ML.TestFramework; +using Xunit; +using Xunit.Abstractions; +using Microsoft.ML.AutoML.CodeGen; + +namespace Microsoft.ML.AutoML.Test +{ + public class TunerTests : BaseTestClass + { + public TunerTests(ITestOutputHelper output) + : base(output) + { + } + + [Fact] + public void CFO_e2e_test() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues)); + for (int i = 0; i != 1000; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + + var param = cfo.Propose(trialSettings); + var option = param.AsType(); + + option.L1Regularization.Should().BeInRange(0.03125f, 32768.0f); + option.L2Regularization.Should().BeInRange(0.03125f, 32768.0f); + + cfo.Update(new TrialResult() + { + DurationInMilliseconds = i * 1000, + Metric = i, + TrialSettings = trialSettings, + }); + } + } + + [Fact] + public void CFO_should_start_from_init_point_if_provided() + { + var trialSettings = new TrialSettings() + { + TrialId = 0, + }; + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + + (x * x + y * y + z * z).Should().Be(0); + } + + [Fact] + public void CFO_should_find_maximum_value_when_function_is_convex() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), false); + double bestMetric = 0; + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = 0, + }; + + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + var metric = this.LSE3D(x, y, z); + bestMetric = Math.Max(bestMetric, metric); + this.Output.WriteLine($"{i} x: {x} y: {y} z: {z}"); + if (x == 10 && y == 10 && z == 10) + { + break; + } + cfo.Update(new TrialResult() + { + DurationInMilliseconds = 1 * 1000, + Metric = metric, + TrialSettings = trialSettings, + }); + } + + bestMetric.Should().BeGreaterThan(this.LSE3D(10, 10, 10) - 2); + } + + [Fact] + public void CFO_should_find_minimum_value_when_function_is_convex() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + double loss = 0; + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + loss = this.LSE3D(x, y, z); + this.Output.WriteLine(loss.ToString()); + this.Output.WriteLine($"{i} x: {x} y: {y} z: {z}"); + + if (x == -10 && y == -10 && z == -10) + { + break; + } + + cfo.Update(new TrialResult() + { + DurationInMilliseconds = 1000, + Metric = loss, + TrialSettings = trialSettings, + }); + } + + loss.Should().BeLessThan(this.LSE3D(-10, -10, -10) + 2); + } + + [Fact] + public void CFO_should_find_minimum_value_when_function_is_F1() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + double bestMetric = 0; + for (int i = 0; i != 1000; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + var param = cfo.Propose(trialSettings).AsType(); + var x = param.X; + var y = param.Y; + var z = param.Z; + var metric = this.F1(x, y, z); + bestMetric = Math.Min(bestMetric, metric); + this.Output.WriteLine($"{i} x: {x} y: {y} z: {z}"); + + if (x == -1 && y == 1 && z == 0) + { + break; + } + + cfo.Update(new TrialResult() + { + DurationInMilliseconds = 1, + Metric = metric, + TrialSettings = trialSettings, + }); + } + + bestMetric.Should().BeLessThan(this.F1(-1, 1, 0) + 2); + } + + [Fact] + public void Hyper_parameters_from_CFO_should_be_culture_invariant_string() + { + var searchSpace = new SearchSpace(); + var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); + var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + var originalCuture = Thread.CurrentThread.CurrentCulture; + var usCulture = new CultureInfo("en-US", false); + Thread.CurrentThread.CurrentCulture = usCulture; + + Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be("."); + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + var param = cfo.Propose(trialSettings).AsType(); + param.X.Should().BeInRange(-10, 10); + } + + var frCulture = new CultureInfo("fr-FR", false); + Thread.CurrentThread.CurrentCulture = frCulture; + Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be(","); + for (int i = 0; i != 100; ++i) + { + var trialSettings = new TrialSettings() + { + TrialId = i, + }; + var param = cfo.Propose(trialSettings).AsType(); + param.X.Should().BeInRange(-10, 10); + } + + Thread.CurrentThread.CurrentCulture = originalCuture; + } + + /// + /// LSE is strictly convex, use this as test object function. + /// + /// + /// + /// + /// + private double LSE3D(double x, double y, double z) + { + return Math.Log(Math.Exp(x) + Math.Exp(y) + Math.Exp(z)); + } + + private double F1(double x, double y, double z) + { + return x * x + 2 * x + 1 + y * y - 2 * y + 1 + z * z; + } + + private class LSE3DSearchSpace + { + [Range(-10.0, 10.0, 0.0, false)] + public double X { get; set; } + + [Range(-10.0, 10.0, 0.0, false)] + public double Y { get; set; } + + [Range(-10.0, 10.0, 0.0, false)] + public double Z { get; set; } + } + } +} From 735264fb7edbd607d6d10e509696ba6b5c377c20 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 28 Mar 2022 15:35:30 -0700 Subject: [PATCH 12/25] Update Microsoft.ML.AutoML.Samples.csproj --- .../Microsoft.ML.AutoML.Samples.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj index 1c4da82682..9d511df6a5 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj @@ -10,7 +10,6 @@ - From 311b8102818ccfb3a5b7bc418454bd3d094692ec Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 28 Mar 2022 15:35:42 -0700 Subject: [PATCH 13/25] Update Program.cs --- docs/samples/Microsoft.ML.AutoML.Samples/Program.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs index 6ecc9663c9..4342873c64 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Program.cs @@ -8,8 +8,6 @@ public static void Main(string[] args) { try { - TitanicExperiment.Run(); - RecommendationExperiment.Run(); Console.Clear(); From 002c083d99fd81f6f28b798a96a145f3f5ee811b Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 28 Mar 2022 15:36:12 -0700 Subject: [PATCH 14/25] Update Assembly.cs --- src/Microsoft.ML.AutoML/Assembly.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Assembly.cs b/src/Microsoft.ML.AutoML/Assembly.cs index 3d1496cead..e3b32b6bf7 100644 --- a/src/Microsoft.ML.AutoML/Assembly.cs +++ b/src/Microsoft.ML.AutoML/Assembly.cs @@ -14,4 +14,3 @@ [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService.Gpu, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.ML.AutoML.Samples, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] From 6301ae23375a21ffc17945a9b65707bda716895d Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 28 Mar 2022 15:39:47 -0700 Subject: [PATCH 15/25] rm unnecessary change --- ...delPipeline_search_space_test.approved.txt | 0 ...eEstimatorAndMultiClassifiers.received.txt | 90 ------------------- ...orPipelineAndMultiClassifiers.received.txt | 88 ------------------ .../SweepableEstimatorPipelineTest.cs | 12 ++- .../SweepableExtensionTest.cs | 2 +- 5 files changed, 12 insertions(+), 180 deletions(-) delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt deleted file mode 100644 index ada341d69d..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt +++ /dev/null @@ -1,90 +0,0 @@ -{ - "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", - "estimators": { - "e0": { - "estimatorType": "FastForestBinary", - "parameter": { - "NumberOfTrees": 4, - "NumberOfLeaves": 4, - "FeatureFraction": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Feature" - } - }, - "e1": { - "estimatorType": "FastTreeOva", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "NumberOfTrees": 4, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e2": { - "estimatorType": "FastForestOva", - "parameter": { - "NumberOfTrees": 4, - "NumberOfLeaves": 4, - "FeatureFraction": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e3": { - "estimatorType": "LightGbmMulti", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "LearningRate": 1, - "NumberOfTrees": 4, - "SubsampleFraction": 1, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e4": { - "estimatorType": "LbfgsLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e5": { - "estimatorType": "LbfgsMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e6": { - "estimatorType": "SdcaMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e7": { - "estimatorType": "SdcaLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt deleted file mode 100644 index adcaa9ec62..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt +++ /dev/null @@ -1,88 +0,0 @@ -{ - "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", - "estimators": { - "e0": { - "estimatorType": "Unknown", - "parameter": {} - }, - "e1": { - "estimatorType": "FeaturizeText", - "parameter": {} - }, - "e2": { - "estimatorType": "FastTreeOva", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "NumberOfTrees": 4, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e3": { - "estimatorType": "FastForestOva", - "parameter": { - "NumberOfTrees": 4, - "NumberOfLeaves": 4, - "FeatureFraction": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e4": { - "estimatorType": "LightGbmMulti", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "LearningRate": 1, - "NumberOfTrees": 4, - "SubsampleFraction": 1, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e5": { - "estimatorType": "LbfgsLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e6": { - "estimatorType": "LbfgsMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e7": { - "estimatorType": "SdcaMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e8": { - "estimatorType": "SdcaLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 76195bdf7d..1f78fd5126 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -15,6 +15,7 @@ using Xunit.Abstractions; using Microsoft.ML.AutoML.CodeGen; using System.Text.Json; +using System.Text.Json.Serialization; namespace Microsoft.ML.AutoML.Test { @@ -28,7 +29,16 @@ public SweepableEstimatorPipelineTest(ITestOutputHelper output) this._jsonSerializerOptions = new JsonSerializerOptions() { WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(), new DoubleToDecimalConverter(), new FloatToDecimalConverter(), + }, }; + + if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) + { + Approvals.UseAssemblyLocationForApprovedFiles(); + } } [Fact] @@ -103,8 +113,8 @@ public void SweepableEstimatorPipeline_can_be_created_from_MultiModelPipeline() } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] + [UseApprovalSubdirectory("ApprovalTests")] public void SweepableEstimatorPipeline_search_space_init_value_test() { var singleModelPipeline = this.CreateSweepbaleEstimatorPipeline(); diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index 62db6c00e4..9b93869672 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -84,8 +84,8 @@ public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorT } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] + [UseApprovalSubdirectory("ApprovalTests")] public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() { var context = new MLContext(); From f7c0a685fe5a6876b753f50157efc48082c8f96f Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 29 Mar 2022 12:31:39 -0700 Subject: [PATCH 16/25] add regression and multi-classification runner --- .../AutoMLExperiment/AutoMLExperiment.cs | 11 +- .../AutoMLExperiment/TrialRunner.cs | 215 +++++++++++++++++- .../AutoMLExperiment/TrialRunnerFactory.cs | 4 + 3 files changed, 217 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index bea18e723e..0d7f8b6dd2 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -3,17 +3,12 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.ML.Data; using Microsoft.ML.Runtime; -using Microsoft.ML.SearchSpace; -using Microsoft.ML.SearchSpace.Tuner; namespace Microsoft.ML.AutoML { @@ -41,6 +36,10 @@ private void InitializeServiceCollection() this._serviceCollection.TryAddSingleton(); this._serviceCollection.TryAddTransient(); this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); + this._serviceCollection.TryAddTransient(); this._serviceCollection.TryAddScoped(); this._serviceCollection.TryAddScoped(); } @@ -210,7 +209,7 @@ private async Task RunAsync(CancellationToken ct) setting = hyperParameterProposer.Propose(setting); monitor.ReportRunningTrial(setting); var runner = runnerFactory.CreateTrialRunner(setting); - var trialResult = runner.Run(this._context, setting); + var trialResult = runner.Run(setting); monitor.ReportCompletedTrial(trialResult); hyperParameterProposer.Update(setting, trialResult); pipelineProposer.Update(setting, trialResult); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index 6cfa7c4781..e682d12939 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -9,12 +9,18 @@ namespace Microsoft.ML.AutoML { internal interface ITrialRunner { - TrialResult Run(MLContext context, TrialSettings settings); + TrialResult Run(TrialSettings settings); } internal class BinaryClassificationCVRunner : ITrialRunner { - public TrialResult Run(MLContext context, TrialSettings settings) + private readonly MLContext _context; + public BinaryClassificationCVRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings @@ -24,8 +30,8 @@ public TrialResult Run(MLContext context, TrialSettings settings) stopWatch.Start(); var fold = datasetSettings.Fold ?? 5; - var pipeline = settings.Pipeline.BuildTrainingPipeline(context, settings.Parameter); - var metrics = context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn); + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var metrics = this._context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[rnd.Next(fold)]; @@ -57,7 +63,13 @@ public TrialResult Run(MLContext context, TrialSettings settings) internal class BinaryClassificationTrainTestRunner : ITrialRunner { - public TrialResult Run(MLContext context, TrialSettings settings) + private readonly MLContext _context; + public BinaryClassificationTrainTestRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) { var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings @@ -66,10 +78,10 @@ public TrialResult Run(MLContext context, TrialSettings settings) var stopWatch = new Stopwatch(); stopWatch.Start(); - var pipeline = settings.Pipeline.BuildTrainingPipeline(context, settings.Parameter); + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); var model = pipeline.Fit(datasetSettings.TrainDataset); var eval = model.Transform(datasetSettings.TestDataset); - var metrics = context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.PredictedColumn, predictedLabelColumnName: metricSettings.TruthColumn); + var metrics = this._context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.PredictedColumn, predictedLabelColumnName: metricSettings.TruthColumn); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var metric = metricSettings.Metric switch @@ -96,4 +108,193 @@ public TrialResult Run(MLContext context, TrialSettings settings) throw new ArgumentException(); } } + + internal class MultiClassificationTrainTestRunner : ITrialRunner + { + private readonly MLContext _context; + public MultiClassificationTrainTestRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = this._context.MulticlassClassification.Evaluate(eval, metricSettings.PredictedColumn, predictedLabelColumnName: metricSettings.TruthColumn); + + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var metric = metricSettings.Metric switch + { + MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, + MulticlassClassificationMetric.MacroAccuracy => metrics.MacroAccuracy, + MulticlassClassificationMetric.TopKAccuracy => metrics.TopKAccuracy, + MulticlassClassificationMetric.LogLoss => metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => metrics.LogLossReduction, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class MultiClassificationCVRunner : ITrialRunner + { + private readonly MLContext _context; + public MultiClassificationCVRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is MultiClassMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var metrics = this._context.MulticlassClassification.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn, seed: settings.ExperimentSettings?.Seed); + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + MulticlassClassificationMetric.MicroAccuracy => res.Metrics.MicroAccuracy, + MulticlassClassificationMetric.MacroAccuracy => res.Metrics.MacroAccuracy, + MulticlassClassificationMetric.TopKAccuracy => res.Metrics.TopKAccuracy, + MulticlassClassificationMetric.LogLoss => res.Metrics.LogLoss, + MulticlassClassificationMetric.LogLossReduction => res.Metrics.LogLossReduction, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class RegressionTrainTestRunner : ITrialRunner + { + private readonly MLContext _context; + public RegressionTrainTestRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + if (settings.ExperimentSettings.DatasetSettings is TrainTestDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var model = pipeline.Fit(datasetSettings.TrainDataset); + var eval = model.Transform(datasetSettings.TestDataset); + var metrics = this._context.Regression.Evaluate(eval, metricSettings.PredictedColumn, scoreColumnName: metricSettings.TruthColumn); + + var metric = metricSettings.Metric switch + { + RegressionMetric.RootMeanSquaredError => metrics.RootMeanSquaredError, + RegressionMetric.RSquared => metrics.RSquared, + RegressionMetric.MeanSquaredError => metrics.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => metrics.MeanAbsoluteError, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } + + internal class RegressionCVRunner : ITrialRunner + { + private readonly MLContext _context; + public RegressionCVRunner(MLContext context) + { + this._context = context; + } + + public TrialResult Run(TrialSettings settings) + { + var rnd = new Random(settings.ExperimentSettings.Seed ?? 0); + if (settings.ExperimentSettings.DatasetSettings is CrossValidateDatasetSettings datasetSettings + && settings.ExperimentSettings.EvaluateMetric is RegressionMetricSettings metricSettings) + { + var stopWatch = new Stopwatch(); + stopWatch.Start(); + var fold = datasetSettings.Fold ?? 5; + + var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); + var metrics = this._context.Regression.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn, seed: settings.ExperimentSettings?.Seed); + // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. + var res = metrics[rnd.Next(fold)]; + var model = res.Model; + var metric = metricSettings.Metric switch + { + RegressionMetric.RootMeanSquaredError => res.Metrics.RootMeanSquaredError, + RegressionMetric.RSquared => res.Metrics.RSquared, + RegressionMetric.MeanSquaredError => res.Metrics.MeanSquaredError, + RegressionMetric.MeanAbsoluteError => res.Metrics.MeanAbsoluteError, + _ => throw new NotImplementedException($"{metricSettings.Metric} is not supported!"), + }; + + stopWatch.Stop(); + + return new TrialResult() + { + Metric = metric, + Model = model, + TrialSettings = settings, + DurationInMilliseconds = stopWatch.ElapsedMilliseconds, + }; + } + + throw new ArgumentException(); + } + } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs index 38d954e8fa..e4ed5f293c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunnerFactory.cs @@ -30,6 +30,10 @@ public TrialRunnerFactory(IServiceProvider provider) { (CrossValidateDatasetSettings, BinaryMetricSettings) => this._provider.GetService(), (TrainTestDatasetSettings, BinaryMetricSettings) => this._provider.GetService(), + (CrossValidateDatasetSettings, MultiClassMetricSettings) => this._provider.GetService(), + (TrainTestDatasetSettings, MultiClassMetricSettings) => this._provider.GetService(), + (CrossValidateDatasetSettings, RegressionMetricSettings) => this._provider.GetService(), + (TrainTestDatasetSettings, RegressionMetricSettings) => this._provider.GetService(), _ => throw new NotImplementedException(), }; From fbccf2a8e90ddc229560e7acc831a93a3f5f394e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 29 Mar 2022 15:23:13 -0700 Subject: [PATCH 17/25] fix bug --- .../AutoMLExperiment/AutoMLExperiment.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 0d7f8b6dd2..dfc020f05c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -126,7 +126,14 @@ public AutoMLExperiment SetTrialRunnerFactory(ITrialRunnerFactory factory) public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) { - this._settings.Pipeline = new MultiModelPipeline().Append(pipeline.Estimators.ToArray()); + var res = new MultiModelPipeline(); + foreach (var e in pipeline.Estimators) + { + res = res.Append(e); + } + + this.SetPipeline(res); + return this; } From 7de0768d2eaf4bac59b358f0848d7bbdcef05aec Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 4 Apr 2022 11:17:38 -0700 Subject: [PATCH 18/25] update IMetricSettings --- .../AutoMLExperiment/IMetricSettings.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs index 8a23c6886e..34b35ba3a5 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs @@ -46,6 +46,11 @@ internal class MultiClassMetricSettings : IMetricSettings public bool IsMaximize => this.Metric switch { + MulticlassClassificationMetric.MacroAccuracy => true, + MulticlassClassificationMetric.MicroAccuracy => true, + MulticlassClassificationMetric.LogLoss => false, + MulticlassClassificationMetric.LogLossReduction => false, + MulticlassClassificationMetric.TopKAccuracy => true, _ => throw new NotImplementedException(), }; } @@ -60,6 +65,10 @@ internal class RegressionMetricSettings : IMetricSettings public bool IsMaximize => this.Metric switch { + RegressionMetric.RSquared => true, + RegressionMetric.RootMeanSquaredError => false, + RegressionMetric.MeanSquaredError => false, + RegressionMetric.MeanAbsoluteError => false, _ => throw new NotImplementedException(), }; } From 8fc794b826a4f54038d4d89853e802b3d85224f8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 4 Apr 2022 11:18:56 -0700 Subject: [PATCH 19/25] checkout assembly --- src/Microsoft.ML.AutoML/Assembly.cs | 1 + src/Microsoft.ML.SearchSpace/Assembly.cs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Assembly.cs b/src/Microsoft.ML.AutoML/Assembly.cs index e3b32b6bf7..c669817f47 100644 --- a/src/Microsoft.ML.AutoML/Assembly.cs +++ b/src/Microsoft.ML.AutoML/Assembly.cs @@ -14,3 +14,4 @@ [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService.Gpu, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder.AutoMLService.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] + diff --git a/src/Microsoft.ML.SearchSpace/Assembly.cs b/src/Microsoft.ML.SearchSpace/Assembly.cs index db60de65f1..39b50da899 100644 --- a/src/Microsoft.ML.SearchSpace/Assembly.cs +++ b/src/Microsoft.ML.SearchSpace/Assembly.cs @@ -7,5 +7,4 @@ [assembly: InternalsVisibleTo("Microsoft.ML.AutoML.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] [assembly: InternalsVisibleTo("Microsoft.ML.SearchSpace.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] [assembly: InternalsVisibleTo("Microsoft.ML.AutoML, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] -[assembly: InternalsVisibleTo("Microsoft.ML.AutoML.Samples, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")] From b3dfbf7e5a308b354375171606a978815921291c Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 5 Apr 2022 10:50:57 -0700 Subject: [PATCH 20/25] fix comments --- .../AutoMLExperiment/AutoMLExperiment.cs | 26 ++++++++++++------- .../AutoMLExperiment/IMetricSettings.cs | 8 +++--- .../AutoMLExperiment/TrialResult.cs | 2 +- .../AutoMLExperiment/TrialRunner.cs | 13 +++++----- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index dfc020f05c..44973c5db9 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ML.Runtime; +using static Microsoft.ML.DataOperationsCatalog; namespace Microsoft.ML.AutoML { @@ -44,9 +45,9 @@ private void InitializeServiceCollection() this._serviceCollection.TryAddScoped(); } - public AutoMLExperiment SetTrainingTimeInSeconds(int trainingTimeInSeconds) + public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) { - this._settings.MaxExperimentTimeInSeconds = (uint)trainingTimeInSeconds; + this._settings.MaxExperimentTimeInSeconds = trainingTimeInSeconds; return this; } @@ -61,6 +62,13 @@ public AutoMLExperiment SetDataset(IDataView train, IDataView test) return this; } + public AutoMLExperiment SetDataset(TrainTestData trainTestSplit) + { + this.SetDataset(trainTestSplit.TrainSet, trainTestSplit.TestSet); + + return this; + } + public AutoMLExperiment SetDataset(IDataView dataset, int fold = 10) { this._settings.DatasetSettings = new CrossValidateDatasetSettings() @@ -137,37 +145,37 @@ public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) return this; } - public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") + public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted", ) { this._settings.EvaluateMetric = new BinaryMetricSettings() { Metric = metric, PredictedColumn = predictedColumn, - TruthColumn = truthColumn, + LabelColumn = labelColumn, }; return this; } - public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") + public AutoMLExperiment SetEvaluateMetric(MulticlassClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") { this._settings.EvaluateMetric = new MultiClassMetricSettings() { Metric = metric, PredictedColumn = predictedColumn, - TruthColumn = truthColumn, + LabelColumn = labelColumn, }; return this; } - public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string predictedColumn = "Predicted", string truthColumn = "label") + public AutoMLExperiment SetEvaluateMetric(RegressionMetric metric, string labelColumn = "label", string scoreColumn = "Score") { this._settings.EvaluateMetric = new RegressionMetricSettings() { Metric = metric, - PredictedColumn = predictedColumn, - TruthColumn = truthColumn, + ScoreColumn = scoreColumn, + LabelColumn = labelColumn, }; return this; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs index 34b35ba3a5..50c2ef6225 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMetricSettings.cs @@ -20,7 +20,7 @@ internal class BinaryMetricSettings : IMetricSettings public string PredictedColumn { get; set; } - public string TruthColumn { get; set; } + public string LabelColumn { get; set; } public bool IsMaximize => this.Metric switch { @@ -42,7 +42,7 @@ internal class MultiClassMetricSettings : IMetricSettings public string PredictedColumn { get; set; } - public string TruthColumn { get; set; } + public string LabelColumn { get; set; } public bool IsMaximize => this.Metric switch { @@ -59,9 +59,9 @@ internal class RegressionMetricSettings : IMetricSettings { public RegressionMetric Metric { get; set; } - public string PredictedColumn { get; set; } + public string ScoreColumn { get; set; } - public string TruthColumn { get; set; } + public string LabelColumn { get; set; } public bool IsMaximize => this.Metric switch { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index 97a93df44b..4b0afab04d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -16,6 +16,6 @@ internal class TrialResult public double Metric { get; set; } - public float DurationInMilliseconds { get; set; } + public double DurationInMilliseconds { get; set; } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs index e682d12939..99a29361df 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialRunner.cs @@ -31,7 +31,7 @@ public TrialResult Run(TrialSettings settings) var fold = datasetSettings.Fold ?? 5; var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); - var metrics = this._context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn); + var metrics = this._context.BinaryClassification.CrossValidateNonCalibrated(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[rnd.Next(fold)]; @@ -81,7 +81,7 @@ public TrialResult Run(TrialSettings settings) var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); var model = pipeline.Fit(datasetSettings.TrainDataset); var eval = model.Transform(datasetSettings.TestDataset); - var metrics = this._context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.PredictedColumn, predictedLabelColumnName: metricSettings.TruthColumn); + var metrics = this._context.BinaryClassification.EvaluateNonCalibrated(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var metric = metricSettings.Metric switch @@ -128,9 +128,8 @@ public TrialResult Run(TrialSettings settings) var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); var model = pipeline.Fit(datasetSettings.TrainDataset); var eval = model.Transform(datasetSettings.TestDataset); - var metrics = this._context.MulticlassClassification.Evaluate(eval, metricSettings.PredictedColumn, predictedLabelColumnName: metricSettings.TruthColumn); + var metrics = this._context.MulticlassClassification.Evaluate(eval, metricSettings.LabelColumn, predictedLabelColumnName: metricSettings.PredictedColumn); - // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var metric = metricSettings.Metric switch { MulticlassClassificationMetric.MicroAccuracy => metrics.MicroAccuracy, @@ -176,7 +175,7 @@ public TrialResult Run(TrialSettings settings) var fold = datasetSettings.Fold ?? 5; var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); - var metrics = this._context.MulticlassClassification.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn, seed: settings.ExperimentSettings?.Seed); + var metrics = this._context.MulticlassClassification.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[rnd.Next(fold)]; var model = res.Model; @@ -224,7 +223,7 @@ public TrialResult Run(TrialSettings settings) var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); var model = pipeline.Fit(datasetSettings.TrainDataset); var eval = model.Transform(datasetSettings.TestDataset); - var metrics = this._context.Regression.Evaluate(eval, metricSettings.PredictedColumn, scoreColumnName: metricSettings.TruthColumn); + var metrics = this._context.Regression.Evaluate(eval, metricSettings.LabelColumn, scoreColumnName: metricSettings.ScoreColumn); var metric = metricSettings.Metric switch { @@ -270,7 +269,7 @@ public TrialResult Run(TrialSettings settings) var fold = datasetSettings.Fold ?? 5; var pipeline = settings.Pipeline.BuildTrainingPipeline(this._context, settings.Parameter); - var metrics = this._context.Regression.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.PredictedColumn, seed: settings.ExperimentSettings?.Seed); + var metrics = this._context.Regression.CrossValidate(datasetSettings.Dataset, pipeline, fold, metricSettings.LabelColumn, seed: settings.ExperimentSettings?.Seed); // now we just randomly pick a model, but a better way is to provide option to pick a model which score is the cloest to average or the best. var res = metrics[rnd.Next(fold)]; var model = res.Model; From 17e7b93e9b994430b2a4f7015c107d6916396d7c Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 7 Apr 2022 11:38:57 -0700 Subject: [PATCH 21/25] fix comments and build --- eng/Versions.props | 1 + .../AutoMLExperiment/AutoMLExperiment.cs | 2 +- .../TrialSettingsProposer/PipelineProposer.cs | 2 -- .../AutoMLExperiment/TunerFactory.cs | 2 +- src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj | 2 +- .../Tuner/{CfoTuner.cs => CostFrugalTuner.cs} | 5 +++-- src/Microsoft.ML.AutoML/Tuner/Flow2.cs | 9 +++++---- test/Microsoft.ML.AutoML.Tests/TunerTests.cs | 12 ++++++------ 8 files changed, 18 insertions(+), 17 deletions(-) rename src/Microsoft.ML.AutoML/Tuner/{CfoTuner.cs => CostFrugalTuner.cs} (93%) diff --git a/eng/Versions.props b/eng/Versions.props index e890cad918..63266131d8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -91,6 +91,7 @@ 5.3.0.1 2.3.0 9.0.1 + 6.0.0 6.0.1 4.4.0 5.6.0-preview.2.6489 diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 44973c5db9..73027a9d5e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -145,7 +145,7 @@ public AutoMLExperiment SetPipeline(SweepableEstimatorPipeline pipeline) return this; } - public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted", ) + public AutoMLExperiment SetEvaluateMetric(BinaryClassificationMetric metric, string labelColumn = "label", string predictedColumn = "Predicted") { this._settings.EvaluateMetric = new BinaryMetricSettings() { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index d0b81f39a2..967497001d 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -67,8 +67,6 @@ public PipelineProposer(AutoMLExperimentSettings settings) }; this._rand = new Random(settings.Seed ?? 0); - // TODO - // use MultiModelPipeline from training configuration if possible. this._multiModelPipeline = null; } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index e3823fcbc9..7ee0111b67 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -31,7 +31,7 @@ public ITuner CreateTuner(TrialSettings settings) var initParameter = settings.Pipeline.Parameter; var isMaximize = experimentSetting.EvaluateMetric.IsMaximize; - return new CfoTuner(searchSpace, initParameter, !isMaximize); + return new CostFrugalTuner(searchSpace, initParameter, !isMaximize); } } diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 2f3cf4b23c..36a14ec5ce 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -20,7 +20,7 @@ true - + diff --git a/src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs similarity index 93% rename from src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs rename to src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs index 2cf6ea7a43..437039abb8 100644 --- a/src/Microsoft.ML.AutoML/Tuner/CfoTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs @@ -8,7 +8,8 @@ namespace Microsoft.ML.AutoML { - internal class CfoTuner : ITuner + // an implemetation of "Frugal Optimization for Cost-related Hyperparameters" : https://www.aaai.org/AAAI21Papers/AAAI-10128.WuQ.pdf + internal class CostFrugalTuner : ITuner { private readonly RandomNumberGenerator _rng = new RandomNumberGenerator(); private readonly SearchSpace.SearchSpace _searchSpace; @@ -24,7 +25,7 @@ internal class CfoTuner : ITuner private bool _initUsed = false; private double _bestMetric; - public CfoTuner(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true) + public CostFrugalTuner(SearchSpace.SearchSpace searchSpace, Parameter initValue = null, bool minimizeMode = true) { this._searchSpace = searchSpace; this._minimize = minimizeMode; diff --git a/src/Microsoft.ML.AutoML/Tuner/Flow2.cs b/src/Microsoft.ML.AutoML/Tuner/Flow2.cs index 109d5d1cce..2ae9c098d7 100644 --- a/src/Microsoft.ML.AutoML/Tuner/Flow2.cs +++ b/src/Microsoft.ML.AutoML/Tuner/Flow2.cs @@ -11,6 +11,9 @@ namespace Microsoft.ML.AutoML { + /// + /// An implementation of Flow2 from https://www.aaai.org/AAAI21Papers/AAAI-10128.WuQ.pdf + /// internal class Flow2 { private const double _stepSize = 0.1; @@ -100,11 +103,9 @@ public Parameter Suggest(int trialId) public void ReceiveTrialResult(int trialId, double metric, double cost) { this._trialCount += 1; - double obj = metric; // flipped in BlendSearch - //double obj = minimize ? metric : -metric; - if (this.BestObj == null || obj < this.BestObj) + if (this.BestObj == null || metric < this.BestObj) { - this.BestObj = obj; + this.BestObj = metric; this._bestConfig = this._configs[trialId]; this._incumbent = this._searchSpace.MappingToFeatureSpace(this._bestConfig); this.CostIncumbent = cost; diff --git a/test/Microsoft.ML.AutoML.Tests/TunerTests.cs b/test/Microsoft.ML.AutoML.Tests/TunerTests.cs index 4a01be897c..fa92a91e83 100644 --- a/test/Microsoft.ML.AutoML.Tests/TunerTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/TunerTests.cs @@ -28,7 +28,7 @@ public void CFO_e2e_test() { var searchSpace = new SearchSpace(); var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues)); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues)); for (int i = 0; i != 1000; ++i) { var trialSettings = new TrialSettings() @@ -60,7 +60,7 @@ public void CFO_should_start_from_init_point_if_provided() }; var searchSpace = new SearchSpace(); var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); var param = cfo.Propose(trialSettings).AsType(); var x = param.X; var y = param.Y; @@ -74,7 +74,7 @@ public void CFO_should_find_maximum_value_when_function_is_convex() { var searchSpace = new SearchSpace(); var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), false); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), false); double bestMetric = 0; for (int i = 0; i != 100; ++i) { @@ -110,7 +110,7 @@ public void CFO_should_find_minimum_value_when_function_is_convex() { var searchSpace = new SearchSpace(); var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); double loss = 0; for (int i = 0; i != 100; ++i) { @@ -148,7 +148,7 @@ public void CFO_should_find_minimum_value_when_function_is_F1() { var searchSpace = new SearchSpace(); var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); double bestMetric = 0; for (int i = 0; i != 1000; ++i) { @@ -185,7 +185,7 @@ public void Hyper_parameters_from_CFO_should_be_culture_invariant_string() { var searchSpace = new SearchSpace(); var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default); - var cfo = new CfoTuner(searchSpace, Parameter.FromObject(initValues), true); + var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true); var originalCuture = Thread.CurrentThread.CurrentCulture; var usCulture = new CultureInfo("en-US", false); Thread.CurrentThread.CurrentCulture = usCulture; From b0822751841599c4985d382113693bf429c5cb1a Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 14 Apr 2022 10:29:36 -0700 Subject: [PATCH 22/25] fix bug --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index dd825ac636..6c0ba66972 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -421,7 +421,7 @@ internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnN fastTreeOption.LabelColumnName = labelColumnName; fastTreeOption.FeatureColumnName = featureColumnName; fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; - res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace(fastTreeOption))); } if (useFastForest) From f2320ffadc942c6670699d45b0a8d8c6b95dfc3c Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 19 Apr 2022 14:57:15 -0700 Subject: [PATCH 23/25] update --- .../AutoMLExperiment/AutoMLExperiment.cs | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 73027a9d5e..c0c3e1a574 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -209,32 +209,46 @@ private async Task RunAsync(CancellationToken ct) while (true) { - if (ct.IsCancellationRequested) + try { - break; + if (ct.IsCancellationRequested) + { + break; + } + + var setting = new TrialSettings() + { + ExperimentSettings = this._settings, + TrialId = trialNum++, + }; + + setting = pipelineProposer.Propose(setting); + setting = hyperParameterProposer.Propose(setting); + monitor.ReportRunningTrial(setting); + var runner = runnerFactory.CreateTrialRunner(setting); + var trialResult = runner.Run(setting); + monitor.ReportCompletedTrial(trialResult); + hyperParameterProposer.Update(setting, trialResult); + pipelineProposer.Update(setting, trialResult); + + var error = this._settings.EvaluateMetric.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < this._bestError) + { + this._bestTrialResult = trialResult; + this._bestError = error; + monitor.ReportBestTrial(trialResult); + } } - - var setting = new TrialSettings() - { - ExperimentSettings = this._settings, - TrialId = trialNum++, - }; - - setting = pipelineProposer.Propose(setting); - setting = hyperParameterProposer.Propose(setting); - monitor.ReportRunningTrial(setting); - var runner = runnerFactory.CreateTrialRunner(setting); - var trialResult = runner.Run(setting); - monitor.ReportCompletedTrial(trialResult); - hyperParameterProposer.Update(setting, trialResult); - pipelineProposer.Update(setting, trialResult); - - var error = this._settings.EvaluateMetric.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; - if (error < this._bestError) + catch (Exception) { - this._bestTrialResult = trialResult; - this._bestError = error; - monitor.ReportBestTrial(trialResult); + if (ct.IsCancellationRequested) + { + break; + } + else + { + throw; + } } } From 1c85cfa21fb43212e95749c07b1cc3ad75230853 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 19 Apr 2022 14:58:34 -0700 Subject: [PATCH 24/25] add comment to estimator cost --- .../AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs index 967497001d..6f619d0e93 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialSettingsProposer/PipelineProposer.cs @@ -41,6 +41,8 @@ internal class PipelineProposer : ISavableProposer public PipelineProposer(AutoMLExperimentSettings settings) { + // this cost is used to initialize eci when started, the smaller the number, the less cost this trainer will use at start, and more likely it will be + // picked. this._estimatorCost = new Dictionary() { { EstimatorType.LightGbmRegression, 0.788 }, From c1fc20b112f7a16cc34538a6df0a2c8267073221 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 20 Apr 2022 19:00:37 -0700 Subject: [PATCH 25/25] rename cfoTunerFactory to CostFrugalFactory --- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 2 +- src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index c0c3e1a574..5de91cedfd 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -34,7 +34,7 @@ private void InitializeServiceCollection() this._serviceCollection.TryAddSingleton(this._settings); this._serviceCollection.TryAddSingleton(); this._serviceCollection.TryAddSingleton(); - this._serviceCollection.TryAddSingleton(); + this._serviceCollection.TryAddSingleton(); this._serviceCollection.TryAddTransient(); this._serviceCollection.TryAddTransient(); this._serviceCollection.TryAddTransient(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs index 7ee0111b67..1f7201d694 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TunerFactory.cs @@ -15,11 +15,11 @@ internal interface ITunerFactory ITuner CreateTuner(TrialSettings settings); } - internal class CfoTunerFactory : ITunerFactory + internal class CostFrugalTunerFactory : ITunerFactory { private readonly IServiceProvider _provider; - public CfoTunerFactory(IServiceProvider provider) + public CostFrugalTunerFactory(IServiceProvider provider) { this._provider = provider; }