From c15a009f188ed46ec3e26780760b67e7e0d3f290 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 6 Mar 2022 11:37:48 -0800 Subject: [PATCH 01/16] add sweepable estimator generator --- .../CodeGen/code_gen_flag.json | 2 +- ...Microsoft.ML.AutoML.SourceGenerator.csproj | 18 + .../SweepableEstimatorGenerator.cs | 88 ++++ .../Template/SweepableEstimator.cs | 387 ++++++++++++++++++ .../Template/SweepableEstimator.tt | 81 ++++ .../Template/SweepableEstimator_T_.cs | 358 ++++++++++++++++ .../Template/SweepableEstimator_T_.tt | 60 +++ .../Utils.cs | 55 +++ 8 files changed, 1048 insertions(+), 1 deletion(-) create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt diff --git a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json index f85ae7c22d..cddb6933a5 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json +++ b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json @@ -3,5 +3,5 @@ "CodeGenCatalogGenerator": false, "EstimatorTypeGenerator": true, "SearchSpaceGenerator": true, - "SweepableEstimatorGenerator": false + "SweepableEstimatorGenerator": true } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj index 271f5d8214..df17513e14 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj @@ -41,6 +41,16 @@ True SearchSpace.tt + + True + True + SweepableEstimator.tt + + + True + True + SweepableEstimator_T_.tt + @@ -52,6 +62,14 @@ TextTemplatingFilePreprocessor SearchSpace.cs + + TextTemplatingFilePreprocessor + SweepableEstimator.cs + + + TextTemplatingFilePreprocessor + SweepableEstimator_T_.cs + diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs new file mode 100644 index 0000000000..15ab0cf4d2 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorGenerator.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; +using Microsoft.ML.AutoML.SourceGenerator; +using SweepableEstimator = Microsoft.ML.AutoML.SourceGenerator.Template.SweepableEstimator; +using SweepableEstimatorT = Microsoft.ML.AutoML.SourceGenerator.Template.SweepableEstimator_T_; +namespace Microsoft.ML.ModelBuilder.SweepableEstimator.CodeGenerator +{ + [Generator] + public class SweepableEstimatorGenerator : ISourceGenerator + { + private const string SweepableEstimatorAttributeDisplayName = Constant.CodeGeneratorNameSpace + "." + "SweepableEstimatorAttribute"; + + public void Execute(GeneratorExecutionContext context) + { + if (context.AdditionalFiles.Where(f => f.Path.Contains("code_gen_flag.json")).First() is AdditionalText text) + { + var json = text.GetText().ToString(); + var flags = JsonSerializer.Deserialize>(json); + if (flags.TryGetValue(nameof(SweepableEstimatorGenerator), out var res) && res == false) + { + return; + } + } + + var estimators = context.AdditionalFiles.Where(f => f.Path.Contains("trainer-estimators.json") || f.Path.Contains("transformer-estimators.json")) + .SelectMany(file => Utils.GetEstimatorsFromJson(file.GetText().ToString()).Estimators) + .ToArray(); + + var code = estimators.SelectMany(e => e.EstimatorTypes.Select(eType => (e, eType, Utils.CreateEstimatorName(e.FunctionName, eType))) + .Select(x => + { + if (x.e.SearchOption == null) + { + return + (x.Item3, + new AutoML.SourceGenerator.Template.SweepableEstimator() + { + NameSpace = Constant.CodeGeneratorNameSpace, + UsingStatements = x.e.UsingStatements, + ArgumentsList = x.e.ArgumentsList, + ClassName = x.Item3, + FunctionName = x.e.FunctionName, + NugetDependencies = x.e.NugetDependencies, + Type = x.eType, + }.TransformText()); + } + else + { + return + (x.Item3, + new SweepableEstimatorT() + { + NameSpace = Constant.CodeGeneratorNameSpace, + UsingStatements = x.e.UsingStatements, + ArgumentsList = x.e.ArgumentsList, + ClassName = x.Item3, + FunctionName = x.e.FunctionName, + NugetDependencies = x.e.NugetDependencies, + Type = x.eType, + TOption = Utils.ToTitleCase(x.e.SearchOption), + }.TransformText()); + } + })); + + foreach (var c in code) + { + context.AddSource(c.Item1 + ".cs", SourceText.From(c.Item2, Encoding.UTF8)); + } + } + + public void Initialize(GeneratorInitializationContext context) + { + return; + //context.RegisterForPostInitialization(i => i.AddSource(nameof(SweepableEstimatorAttribute), SweepableEstimatorAttribute)); + } + } +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs new file mode 100644 index 0000000000..ceb30b1082 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs @@ -0,0 +1,387 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version: 17.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +namespace Microsoft.ML.AutoML.SourceGenerator.Template +{ + using System.Linq; + using System.Text; + using System.Collections.Generic; + using System; + + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class SweepableEstimator : SweepableEstimatorBase + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write(@" +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; + +namespace "); + this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); + this.Write("\r\n{\r\n internal partial class "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(" : SweepableEstimator\r\n {\r\n public "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write("()\r\n {\r\n this.EstimatorType = EstimatorType."); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(";\r\n }\r\n \r\n"); + foreach(var arg in ArgumentsList){ + var typeAttributeName = Utils.CapitalFirstLetter(arg.ArgumentType); + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); + this.Write(" ["); + this.Write(this.ToStringHelper.ToStringWithCulture(typeAttributeName)); + this.Write("]\r\n [JsonProperty(NullValueHandling=NullValueHandling.Ignore)]\r\n pu" + + "blic string "); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(" { get; set; }\r\n\r\n"); +} + this.Write(" public override IEnumerable CSharpUsingStatements \r\n {\r\n " + + " get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); + this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + + " {\r\n get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); + this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + + " get => \""); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); + this.Write("."); + this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); + this.Write("\";\r\n }\r\n\r\n public override string ToCSharpCode()\r\n {\r\n"); +if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){ + this.Write(" return this.BuildCSharpCodeForTrainers();\r\n"); +}else if (Type == "OneVersusAll"){ + this.Write(" return this.BuildCSharpCodeForOva();\r\n"); +}else{ + this.Write(" return this.BuildCSharpCodeForTransformers();\r\n"); +} + this.Write(" }\r\n\r\n internal override void UpdatePropertiesFromOptions(Dictionar" + + "y options)\r\n {\r\n"); + foreach(var arg in ArgumentsList){ + var typeAttributeName = arg.ArgumentType; + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); + if(typeAttributeName == "integer"){ + this.Write(" this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(" = options.ContainsKey(nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write("))? this.ToIntegerString(options[nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(")]) : this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(";\r\n "); +}else{ + this.Write(" this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(" = options.ContainsKey(nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write("))? options[nameof(this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(")] : this."); + this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); + this.Write(";\r\n "); +}} + this.Write(" }\r\n }\r\n}\r\n\r\n"); + return this.GenerationEnvironment.ToString(); + } + +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} + + } + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class SweepableEstimatorBase + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + protected System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; + } + set + { + this.generationEnvironmentField = value; + } + } + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + } + return this.errorsField; + } + } + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); + } + return this.indentLengthsField; + } + } + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; + } + } + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; + } + set + { + this.sessionField = value; + } + } + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } + } + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; + } + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); + } + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); + } + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); + } + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; + } + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; + } + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; + } + set + { + if ((value != null)) + { + this.formatProviderField = value; + } + } + } + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { + throw new global::System.ArgumentNullException("objectToConvert"); + } + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); + } + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); + } + } + } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion + } + #endregion +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt new file mode 100644 index 0000000000..b4e9a68558 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt @@ -0,0 +1,81 @@ +<#@ template language="C#" linePragmas="false" visibility = "internal"#> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> + +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; + +namespace <#=NameSpace#> +{ + internal partial class <#=ClassName#> : SweepableEstimator + { + public <#=ClassName#>() + { + this.EstimatorType = EstimatorType.<#=ClassName#>; + } + +<# foreach(var arg in ArgumentsList){ + var typeAttributeName = Utils.CapitalFirstLetter(arg.ArgumentType); + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName);#> + [<#=typeAttributeName#>] + [JsonProperty(NullValueHandling=NullValueHandling.Ignore)] + public string <#=propertyName#> { get; set; } + +<#}#> + public override IEnumerable CSharpUsingStatements + { + get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; + } + + public override IEnumerable NugetDependencies + { + get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; + } + + public override string FunctionName + { + get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; + } + + public override string ToCSharpCode() + { +<#if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){#> + return this.BuildCSharpCodeForTrainers(); +<#}else if (Type == "OneVersusAll"){#> + return this.BuildCSharpCodeForOva(); +<#}else{#> + return this.BuildCSharpCodeForTransformers(); +<#}#> + } + + internal override void UpdatePropertiesFromOptions(Dictionary options) + { +<# foreach(var arg in ArgumentsList){ + var typeAttributeName = arg.ArgumentType; + var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); + if(typeAttributeName == "integer"){#> + this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? this.ToIntegerString(options[nameof(this.<#=propertyName#>)]) : this.<#=propertyName#>; + <#}else{#> + this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? options[nameof(this.<#=propertyName#>)] : this.<#=propertyName#>; + <#}}#> + } + } +} + +<#+ +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} +#> \ No newline at end of file diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs new file mode 100644 index 0000000000..9b1327aacb --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs @@ -0,0 +1,358 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version: 17.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +namespace Microsoft.ML.AutoML.SourceGenerator.Template +{ + using System.Linq; + using System.Text; + using System.Collections.Generic; + using System; + + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class SweepableEstimator_T_ : SweepableEstimator_T_Base + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write(@" +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; +using Microsoft.ML.SearchSpace; + +namespace "); + this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); + this.Write("\r\n{\r\n internal partial class "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(" : SweepableEstimator<"); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write(">\r\n {\r\n public "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write("("); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write(" defaultOption, SearchSpace<"); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write("> searchSpace = null)\r\n {\r\n this.Parameter = defaultOption;\r\n " + + " this.SearchSpace = searchSpace;\r\n this.EstimatorType = Esti" + + "matorType."); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(";\r\n }\r\n\r\n internal "); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write("()\r\n {\r\n this.EstimatorType = EstimatorType."); + this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); + this.Write(";\r\n this.Parameter = new "); + this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); + this.Write("();\r\n }\r\n \r\n public override IEnumerable CSharpUsingStat" + + "ements \r\n {\r\n get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); + this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + + " {\r\n get => new string[] {"); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); + this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + + " get => \""); + this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); + this.Write("."); + this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); + this.Write("\";\r\n }\r\n }\r\n}\r\n\r\n"); + return this.GenerationEnvironment.ToString(); + } + +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} +public string TOption {get; set;} + + } + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class SweepableEstimator_T_Base + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + protected System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; + } + set + { + this.generationEnvironmentField = value; + } + } + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + } + return this.errorsField; + } + } + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); + } + return this.indentLengthsField; + } + } + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; + } + } + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; + } + set + { + this.sessionField = value; + } + } + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } + } + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; + } + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); + } + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); + } + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); + } + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; + } + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; + } + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; + } + set + { + if ((value != null)) + { + this.formatProviderField = value; + } + } + } + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { + throw new global::System.ArgumentNullException("objectToConvert"); + } + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); + } + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); + } + } + } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion + } + #endregion +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt new file mode 100644 index 0000000000..ff6b3e5974 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt @@ -0,0 +1,60 @@ +<#@ template language="C#" linePragmas="false" visibility = "internal"#> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> + +using System.Collections.Generic; +using Newtonsoft.Json; +using SweepableEstimator = Microsoft.ML.AutoML.SweepableEstimator; +using Microsoft.ML.AutoML.CodeGen; +using ColorsOrder = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorsOrder; +using ColorBits = Microsoft.ML.Transforms.Image.ImagePixelExtractingEstimator.ColorBits; +using ResizingKind = Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind; +using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor; +using Microsoft.ML.SearchSpace; + +namespace <#=NameSpace#> +{ + internal partial class <#=ClassName#> : SweepableEstimator<<#=TOption#>> + { + public <#=ClassName#>(<#=TOption#> defaultOption, SearchSpace<<#=TOption#>> searchSpace = null) + { + this.Parameter = defaultOption; + this.SearchSpace = searchSpace; + this.EstimatorType = EstimatorType.<#=ClassName#>; + } + + internal <#=ClassName#>() + { + this.EstimatorType = EstimatorType.<#=ClassName#>; + this.Parameter = new <#=TOption#>(); + } + + public override IEnumerable CSharpUsingStatements + { + get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; + } + + public override IEnumerable NugetDependencies + { + get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; + } + + public override string FunctionName + { + get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; + } + } +} + +<#+ +public string NameSpace {get;set;} +public string ClassName {get;set;} +public string FunctionName {get;set;} +public string Type {get;set;} +public IEnumerable ArgumentsList {get;set;} +public IEnumerable UsingStatements {get; set;} +public IEnumerable NugetDependencies {get; set;} +public string TOption {get; set;} +#> \ No newline at end of file diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs index c71e7a0f90..e6897a8d6a 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Utils.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -70,5 +71,59 @@ public static string ToTitleCase(string str) { return string.Join(string.Empty, str.Split('_', ' ', '-').Select(x => CapitalFirstLetter(x))); } + + public static string GetPrefix(string estimatorType) + { + if (estimatorType == "BinaryClassification") + { + return "BinaryClassification.Trainers"; + } + if (estimatorType == "MultiClassification") + { + return "MulticlassClassification.Trainers"; + } + if (estimatorType == "Regression") + { + return "Regression.Trainers"; + } + if (estimatorType == "Ranking") + { + return "Ranking.Trainers"; + } + if (estimatorType == "OneVersusAll") + { + return "BinaryClassification.Trainers"; + } + if (estimatorType == "Recommendation") + { + return "Recommendation().Trainers"; + } + if (estimatorType == "Transforms") + { + return "Transforms"; + } + if (estimatorType == "Categorical") + { + return "Transforms.Categorical"; + } + if (estimatorType == "Conversion") + { + return "Transforms.Conversion"; + } + if (estimatorType == "Text") + { + return "Transforms.Text"; + } + if (estimatorType == "Calibrators") + { + return "BinaryClassification.Calibrators"; + } + if (estimatorType == "Forecasting") + { + return "Forecasting"; + } + + throw new NotImplementedException(); + } } } From 44da68a1a9d143123924322ed7a0987f5c7a92f8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 6 Mar 2022 12:01:44 -0800 Subject: [PATCH 02/16] add runtime --- src/Microsoft.ML.AutoML/AutoMlUtils.cs | 34 +++++++ .../Microsoft.ML.AutoML.csproj | 10 +- .../Estimators/ApplyOnnx.cs | 18 ++++ .../Estimators/Calibrators.cs | 18 ++++ .../Estimators/Concatenate.cs | 14 +++ .../Estimators/FastForest.cs | 63 ++++++++++++ .../SweepableEstimator/Estimators/FastTree.cs | 96 +++++++++++++++++++ .../Estimators/FeaturizeText.cs | 14 +++ .../Estimators/ForecastBySsa.cs | 21 ++++ .../SweepableEstimator/Estimators/Images.cs | 47 +++++++++ .../SweepableEstimator/Estimators/Lbfgs.cs | 81 ++++++++++++++++ .../SweepableEstimator/Estimators/LightGbm.cs | 92 ++++++++++++++++++ .../Estimators/MapValueToKey.cs | 22 +++++ .../Estimators/MatrixFactorization.cs | 14 +++ .../Estimators/NormalizeMinMax.cs | 16 ++++ .../Estimators/OneHotEncoding.cs | 24 +++++ .../Estimators/ReplaceMissingValue.cs | 15 +++ .../SweepableEstimator/Estimators/Sdca.cs | 81 ++++++++++++++++ .../Estimators/TypeConvert.cs | 15 +++ .../SweepableEstimator/SweepableEstimator.cs | 12 --- .../Template/SweepableEstimator.cs | 48 ++-------- .../Template/SweepableEstimator.tt | 29 +----- .../Template/SweepableEstimator_T_.cs | 20 ++-- .../Template/SweepableEstimator_T_.tt | 10 +- 24 files changed, 716 insertions(+), 98 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs diff --git a/src/Microsoft.ML.AutoML/AutoMlUtils.cs b/src/Microsoft.ML.AutoML/AutoMlUtils.cs index 8e9ad11a22..07ee28c6c3 100644 --- a/src/Microsoft.ML.AutoML/AutoMlUtils.cs +++ b/src/Microsoft.ML.AutoML/AutoMlUtils.cs @@ -3,12 +3,46 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Threading; namespace Microsoft.ML.AutoML { internal static class AutoMlUtils { + private const string MLNetMaxThread = "MLNET_MAX_THREAD"; + public static readonly ThreadLocal Random = new ThreadLocal(() => new Random()); + + /// + /// Return number of thread if MLNET_MAX_THREAD is set, otherwise return null. + /// + public static int? GetNumberOfThreadFromEnvrionment() + { + var res = Environment.GetEnvironmentVariable(MLNetMaxThread); + + if (int.TryParse(res, out var numberOfThread)) + { + return numberOfThread; + } + + return null; + } + + public static InputOutputColumnPair[] CreateInputOutputColumnPairsFromStrings(string[] inputs, string[] outputs) + { + if (inputs.Length != outputs.Length) + { + throw new Exception("inputs and outputs count must match"); + } + + var res = new List(); + for (int i = 0; i != inputs.Length; ++i) + { + res.Add(new InputOutputColumnPair(outputs[i], inputs[i])); + } + + return res.ToArray(); + } } } diff --git a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj index 46632e232a..6d79838dff 100644 --- a/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj +++ b/src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj @@ -14,11 +14,13 @@ all + all true + @@ -43,13 +45,13 @@ - + - + - - + + diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs new file mode 100644 index 0000000000..daabd21e6b --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ApplyOnnx.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ApplyOnnxModel + { + public override IEstimator BuildFromOption(MLContext context, ApplyOnnxModelOption param) + { + return context.Transforms.ApplyOnnxModel(outputColumnName: param.OutputColumnName, inputColumnName: param.InputColumnName, modelFile: param.ModelFile, gpuDeviceId: param.GpuDeviceId, fallbackToCpu: param.FallbackToCpu); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs new file mode 100644 index 0000000000..f59c7264de --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Calibrators.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class Naive + { + public override IEstimator BuildFromOption(MLContext context, NaiveOption param) + { + return context.BinaryClassification.Calibrators.Naive(param.LabelColumnName, param.ScoreColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs new file mode 100644 index 0000000000..aa085543af --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Concatenate.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class Concatenate + { + public override IEstimator BuildFromOption(MLContext context, ConcatOption param) + { + return context.Transforms.Concatenate(param.OutputColumnName, param.InputColumnNames); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs new file mode 100644 index 0000000000..c9e900f023 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastForest.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers.FastTree; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class FastForestOva + { + public override IEstimator BuildFromOption(MLContext context, FastForestOption param) + { + var option = new FastForestBinaryTrainer.Options() + { + NumberOfTrees = param.NumberOfTrees, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + FeatureFraction = param.FeatureFraction, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastForest(option), labelColumnName: param.LabelColumnName); + } + } + + internal partial class FastForestRegression + { + public override IEstimator BuildFromOption(MLContext context, FastForestOption param) + { + var option = new FastForestRegressionTrainer.Options() + { + NumberOfTrees = param.NumberOfTrees, + FeatureFraction = param.FeatureFraction, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.Regression.Trainers.FastForest(option); + } + } + + internal partial class FastForestBinary + { + public override IEstimator BuildFromOption(MLContext context, FastForestOption param) + { + var option = new FastForestBinaryTrainer.Options() + { + NumberOfTrees = param.NumberOfTrees, + NumberOfLeaves = param.NumberOfLeaves, + FeatureFraction = param.FeatureFraction, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.BinaryClassification.Trainers.FastForest(option); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs new file mode 100644 index 0000000000..cc6ae89136 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs @@ -0,0 +1,96 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers.FastTree; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class FastTreeOva + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeBinaryTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastTree(option), labelColumnName: param.LabelColumnName); + } + } + + internal partial class FastTreeRegression + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeRegressionTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.Regression.Trainers.FastTree(option); + } + } + + internal partial class FastTreeTweedieRegression + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeTweedieTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.Regression.Trainers.FastTreeTweedie(option); + } + } + + internal partial class FastTreeBinary + { + public override IEstimator BuildFromOption(MLContext context, FastTreeOption param) + { + var option = new FastTreeBinaryTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfTrees = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + FeatureFraction = param.FeatureFraction, + }; + + return context.BinaryClassification.Trainers.FastTree(option); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs new file mode 100644 index 0000000000..67c4905ebf --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FeaturizeText.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class FeaturizeText + { + public override IEstimator BuildFromOption(MLContext context, FeaturizeTextOption param) + { + return context.Transforms.Text.FeaturizeText(param.OutputColumnName, param.InputColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs new file mode 100644 index 0000000000..993ffdaad4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ForecastBySsa.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ForecastBySsa + { + public override IEstimator BuildFromOption(MLContext context, SsaOption param) + { + if (param.SeriesLength <= param.WindowSize || param.TrainSize <= 2 * param.WindowSize) + { + throw new Exception("ForecastBySsa param check error"); + } + + return context.Forecasting.ForecastBySsa(param.OutputColumnName, param.InputColumnName, param.WindowSize, param.SeriesLength, param.TrainSize, param.Horizon, confidenceLowerBoundColumn: param.ConfidenceLowerBoundColumn, confidenceUpperBoundColumn: param.ConfidenceUpperBoundColumn); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs new file mode 100644 index 0000000000..6df898e0f3 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class LoadImages + { + public override IEstimator BuildFromOption(MLContext context, LoadImageOption param) + { + return context.Transforms.LoadImages(param.OutputColumnName, param.ImageFolder, param.InputColumnName); + } + } + + internal partial class LoadRawImageBytes + { + public override IEstimator BuildFromOption(MLContext context, LoadImageOption param) + { + return context.Transforms.LoadRawImageBytes(param.OutputColumnName, param.ImageFolder, param.InputColumnName); + } + } + + internal partial class ResizeImages + { + public override IEstimator BuildFromOption(MLContext context, ResizeImageOption param) + { + return context.Transforms.ResizeImages(param.OutputColumnName, param.ImageWidth, param.ImageHeight, param.InputColumnName, param.Resizing, param.CropAnchor); + } + } + + internal partial class ExtractPixels + { + public override IEstimator BuildFromOption(MLContext context, ExtractPixelsOption param) + { + return context.Transforms.ExtractPixels(param.OutputColumnName, param.InputColumnName, param.ColorsToExtract, param.OrderOfExtraction, outputAsFloatArray: param.OutputAsFloatArray); + } + } + + internal partial class ImageClassificationMulti + { + public override IEstimator BuildFromOption(MLContext context, ImageClassificationOption param) + { + + return context.MulticlassClassification.Trainers.ImageClassification(param.LabelColumnName, param.FeatureColumnName, param.ScoreColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs new file mode 100644 index 0000000000..8480b0e6ee --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Lbfgs.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class LbfgsMaximumEntropyMulti + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsMaximumEntropyMulticlassTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.MulticlassClassification.Trainers.LbfgsMaximumEntropy(option); + } + } + + internal partial class LbfgsPoissonRegressionRegression + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsPoissonRegressionTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.Regression.Trainers.LbfgsPoissonRegression(option); + } + } + + internal partial class LbfgsLogisticRegressionBinary + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsLogisticRegressionBinaryTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.BinaryClassification.Trainers.LbfgsLogisticRegression(option); + } + } + + internal partial class LbfgsLogisticRegressionOva + { + public override IEstimator BuildFromOption(MLContext context, LbfgsOption param) + { + var option = new LbfgsLogisticRegressionBinaryTrainer.Options() + { + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + var binaryTrainer = context.BinaryClassification.Trainers.LbfgsLogisticRegression(option); + return context.MulticlassClassification.Trainers.OneVersusAll(binaryTrainer, param.LabelColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs new file mode 100644 index 0000000000..7164d00907 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/LightGbm.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers.LightGbm; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class LightGbmMulti + { + public override IEstimator BuildFromOption(MLContext context, LgbmOption param) + { + var option = new LightGbmMulticlassTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfIterations = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + Booster = new GradientBooster.Options() + { + SubsampleFraction = param.SubsampleFraction, + FeatureFraction = param.FeatureFraction, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + }, + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + }; + + return context.MulticlassClassification.Trainers.LightGbm(option); + } + } + + internal partial class LightGbmBinary + { + public override IEstimator BuildFromOption(MLContext context, LgbmOption param) + { + var option = new LightGbmBinaryTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfIterations = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + Booster = new GradientBooster.Options() + { + SubsampleFraction = param.SubsampleFraction, + FeatureFraction = param.FeatureFraction, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + }, + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + }; + + return context.BinaryClassification.Trainers.LightGbm(option); + } + } + + internal partial class LightGbmRegression + { + public override IEstimator BuildFromOption(MLContext context, LgbmOption param) + { + var option = new LightGbmRegressionTrainer.Options() + { + NumberOfLeaves = param.NumberOfLeaves, + NumberOfIterations = param.NumberOfTrees, + MinimumExampleCountPerLeaf = param.MinimumExampleCountPerLeaf, + LearningRate = param.LearningRate, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + Booster = new GradientBooster.Options() + { + SubsampleFraction = param.SubsampleFraction, + FeatureFraction = param.FeatureFraction, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + }, + MaximumBinCountPerFeature = param.MaximumBinCountPerFeature, + }; + + return context.Regression.Trainers.LightGbm(option); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs new file mode 100644 index 0000000000..36dbd5de7f --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class MapValueToKey + { + public override IEstimator BuildFromOption(MLContext context, MapValueToKeyOption param) + { + return context.Transforms.Conversion.MapValueToKey(param.OutputColumnName, param.InputColumnName); + } + } + + internal partial class MapKeyToValue + { + public override IEstimator BuildFromOption(MLContext context, MapKeyToValueOption param) + { + return context.Transforms.Conversion.MapKeyToValue(param.OutputColumnName, param.InputColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs new file mode 100644 index 0000000000..91900fd4e0 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MatrixFactorization.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class MatrixFactorization + { + public override IEstimator BuildFromOption(MLContext context, MatrixFactorizationOption param) + { + return context.Recommendation().Trainers.MatrixFactorization(param.LabelColumnName, param.MatrixColumnIndexColumnName, param.MatrixRowIndexColumnName, param.ApproximationRank, param.LearningRate, param.NumberOfIterations); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs new file mode 100644 index 0000000000..73fa584ffb --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeMinMax.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class NormalizeMinMax + { + public override IEstimator BuildFromOption(MLContext context, NormalizeMinMaxOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.OutputColumnNames, param.InputColumnNames); + + return context.Transforms.NormalizeMinMax(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs new file mode 100644 index 0000000000..be5679ca3c --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/OneHotEncoding.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class OneHotEncoding + { + public override IEstimator BuildFromOption(MLContext context, OneHotOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.Categorical.OneHotEncoding(inputOutputPairs); + } + } + + internal partial class OneHotHashEncoding + { + public override IEstimator BuildFromOption(MLContext context, OneHotOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.Categorical.OneHotHashEncoding(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs new file mode 100644 index 0000000000..e638798a88 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/ReplaceMissingValue.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ReplaceMissingValues + { + public override IEstimator BuildFromOption(MLContext context, ReplaceMissingValueOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.ReplaceMissingValues(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs new file mode 100644 index 0000000000..1e073d56e4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Sdca.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Trainers; + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class SdcaRegression + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaRegressionTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.Regression.Trainers.Sdca(option); + } + } + + internal partial class SdcaMaximumEntropyMulti + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaMaximumEntropyMulticlassTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.MulticlassClassification.Trainers.SdcaMaximumEntropy(option); + } + } + + internal partial class SdcaLogisticRegressionBinary + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaLogisticRegressionBinaryTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + return context.BinaryClassification.Trainers.SdcaLogisticRegression(option); + } + } + + internal partial class SdcaLogisticRegressionOva + { + public override IEstimator BuildFromOption(MLContext context, SdcaOption param) + { + var option = new SdcaLogisticRegressionBinaryTrainer.Options() + { + LabelColumnName = param.LabelColumnName, + FeatureColumnName = param.FeatureColumnName, + ExampleWeightColumnName = param.ExampleWeightColumnName, + L1Regularization = param.L1Regularization, + L2Regularization = param.L2Regularization, + NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(), + }; + + var binaryTrainer = context.BinaryClassification.Trainers.SdcaLogisticRegression(option); + return context.MulticlassClassification.Trainers.OneVersusAll(binaryEstimator: binaryTrainer, labelColumnName: param.LabelColumnName); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs new file mode 100644 index 0000000000..f5e567baa0 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.ML.AutoML.CodeGen +{ + internal partial class ConvertType + { + public override IEstimator BuildFromOption(MLContext context, ConvertTypeOption param) + { + var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames); + return context.Transforms.Conversion.ConvertType(inputOutputPairs); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs index db5634258d..d1e65bf898 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs @@ -50,11 +50,6 @@ public virtual IEstimator BuildFromOption(MLContext context, Param internal virtual IEnumerable NugetDependencies { get; } internal virtual string FunctionName { get; } - - internal virtual string ToDisplayString(Parameter param) - { - throw new NotImplementedException(); - } } internal abstract class SweepableEstimator : SweepableEstimator @@ -75,12 +70,5 @@ public override IEstimator BuildFromOption(MLContext context, Para { return this.BuildFromOption(context, param.AsType()); } - - internal abstract string ToDisplayString(TOption param); - - internal override string ToDisplayString(Parameter param) - { - return this.ToDisplayString(param.AsType()); - } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs index ceb30b1082..59d090db8a 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.cs @@ -54,52 +54,18 @@ namespace "); this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); this.Write(" { get; set; }\r\n\r\n"); } - this.Write(" public override IEnumerable CSharpUsingStatements \r\n {\r\n " + - " get => new string[] {"); + this.Write(" internal override IEnumerable CSharpUsingStatements \r\n {\r\n" + + " get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); - this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + - " {\r\n get => new string[] {"); + this.Write("};\r\n }\r\n\r\n internal override IEnumerable NugetDependencies\r" + + "\n {\r\n get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); - this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + - " get => \""); + this.Write("};\r\n }\r\n\r\n internal override string FunctionName \r\n {\r\n " + + " get => \""); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); this.Write("."); this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); - this.Write("\";\r\n }\r\n\r\n public override string ToCSharpCode()\r\n {\r\n"); -if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){ - this.Write(" return this.BuildCSharpCodeForTrainers();\r\n"); -}else if (Type == "OneVersusAll"){ - this.Write(" return this.BuildCSharpCodeForOva();\r\n"); -}else{ - this.Write(" return this.BuildCSharpCodeForTransformers();\r\n"); -} - this.Write(" }\r\n\r\n internal override void UpdatePropertiesFromOptions(Dictionar" + - "y options)\r\n {\r\n"); - foreach(var arg in ArgumentsList){ - var typeAttributeName = arg.ArgumentType; - var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); - if(typeAttributeName == "integer"){ - this.Write(" this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(" = options.ContainsKey(nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write("))? this.ToIntegerString(options[nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(")]) : this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(";\r\n "); -}else{ - this.Write(" this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(" = options.ContainsKey(nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write("))? options[nameof(this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(")] : this."); - this.Write(this.ToStringHelper.ToStringWithCulture(propertyName)); - this.Write(";\r\n "); -}} - this.Write(" }\r\n }\r\n}\r\n\r\n"); + this.Write("\";\r\n }\r\n }\r\n}\r\n\r\n"); return this.GenerationEnvironment.ToString(); } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt index b4e9a68558..fc0ccbb24f 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator.tt @@ -30,43 +30,20 @@ namespace <#=NameSpace#> public string <#=propertyName#> { get; set; } <#}#> - public override IEnumerable CSharpUsingStatements + internal override IEnumerable CSharpUsingStatements { get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; } - public override IEnumerable NugetDependencies + internal override IEnumerable NugetDependencies { get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; } - public override string FunctionName + internal override string FunctionName { get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; } - - public override string ToCSharpCode() - { -<#if (Type == "BinaryClassification" || Type == "MultiClassification" || Type == "Regression" || Type == "Ranking"){#> - return this.BuildCSharpCodeForTrainers(); -<#}else if (Type == "OneVersusAll"){#> - return this.BuildCSharpCodeForOva(); -<#}else{#> - return this.BuildCSharpCodeForTransformers(); -<#}#> - } - - internal override void UpdatePropertiesFromOptions(Dictionary options) - { -<# foreach(var arg in ArgumentsList){ - var typeAttributeName = arg.ArgumentType; - var propertyName = Utils.CapitalFirstLetter(arg.ArgumentName); - if(typeAttributeName == "integer"){#> - this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? this.ToIntegerString(options[nameof(this.<#=propertyName#>)]) : this.<#=propertyName#>; - <#}else{#> - this.<#=propertyName#> = options.ContainsKey(nameof(this.<#=propertyName#>))? options[nameof(this.<#=propertyName#>)] : this.<#=propertyName#>; - <#}}#> - } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs index 9b1327aacb..5db7e76a39 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.cs @@ -48,24 +48,24 @@ namespace "); this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); this.Write(" defaultOption, SearchSpace<"); this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); - this.Write("> searchSpace = null)\r\n {\r\n this.Parameter = defaultOption;\r\n " + - " this.SearchSpace = searchSpace;\r\n this.EstimatorType = Esti" + - "matorType."); + this.Write("> searchSpace = null)\r\n {\r\n this.TParameter = defaultOption;\r\n " + + " this.SearchSpace = searchSpace;\r\n this.EstimatorType = Est" + + "imatorType."); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); this.Write(";\r\n }\r\n\r\n internal "); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); this.Write("()\r\n {\r\n this.EstimatorType = EstimatorType."); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); - this.Write(";\r\n this.Parameter = new "); + this.Write(";\r\n this.TParameter = new "); this.Write(this.ToStringHelper.ToStringWithCulture(TOption)); - this.Write("();\r\n }\r\n \r\n public override IEnumerable CSharpUsingStat" + - "ements \r\n {\r\n get => new string[] {"); + this.Write("();\r\n }\r\n \r\n internal override IEnumerable CSharpUsingSt" + + "atements \r\n {\r\n get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};")))); - this.Write("};\r\n }\r\n\r\n public override IEnumerable NugetDependencies\r\n " + - " {\r\n get => new string[] {"); + this.Write("};\r\n }\r\n\r\n internal override IEnumerable NugetDependencies\r" + + "\n {\r\n get => new string[] {"); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.PrettyPrintListOfString(NugetDependencies))); - this.Write("};\r\n }\r\n\r\n public override string FunctionName \r\n {\r\n " + - " get => \""); + this.Write("};\r\n }\r\n\r\n internal override string FunctionName \r\n {\r\n " + + " get => \""); this.Write(this.ToStringHelper.ToStringWithCulture(Utils.GetPrefix(Type))); this.Write("."); this.Write(this.ToStringHelper.ToStringWithCulture(FunctionName)); diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt index ff6b3e5974..14ee696ea2 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimator_T_.tt @@ -20,7 +20,7 @@ namespace <#=NameSpace#> { public <#=ClassName#>(<#=TOption#> defaultOption, SearchSpace<<#=TOption#>> searchSpace = null) { - this.Parameter = defaultOption; + this.TParameter = defaultOption; this.SearchSpace = searchSpace; this.EstimatorType = EstimatorType.<#=ClassName#>; } @@ -28,20 +28,20 @@ namespace <#=NameSpace#> internal <#=ClassName#>() { this.EstimatorType = EstimatorType.<#=ClassName#>; - this.Parameter = new <#=TOption#>(); + this.TParameter = new <#=TOption#>(); } - public override IEnumerable CSharpUsingStatements + internal override IEnumerable CSharpUsingStatements { get => new string[] {<#=Utils.PrettyPrintListOfString(UsingStatements.Select(x => $"using {x};"))#>}; } - public override IEnumerable NugetDependencies + internal override IEnumerable NugetDependencies { get => new string[] {<#=Utils.PrettyPrintListOfString(NugetDependencies)#>}; } - public override string FunctionName + internal override string FunctionName { get => "<#=Utils.GetPrefix(Type)#>.<#=FunctionName#>"; } From 365dd6a10e166a06ef1d32ff2c9ed3d6aeb51d26 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 6 Mar 2022 12:15:30 -0800 Subject: [PATCH 03/16] enable sweepabe estimator factory generator --- .../CodeGen/code_gen_flag.json | 2 +- ...Microsoft.ML.AutoML.SourceGenerator.csproj | 9 + .../SweepableEstimatorFactoryGenerator.cs | 55 +++ .../Template/SweepableEstimatorFactory.cs | 329 ++++++++++++++++++ .../Template/SweepableEstimatorFactory.tt | 36 ++ 5 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs create mode 100644 tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt diff --git a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json index cddb6933a5..6efed0e19a 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json +++ b/src/Microsoft.ML.AutoML/CodeGen/code_gen_flag.json @@ -1,6 +1,6 @@ { "EstimatorFactoryGenerator": false, - "CodeGenCatalogGenerator": false, + "SweepableEstimatorFactory": true, "EstimatorTypeGenerator": true, "SearchSpaceGenerator": true, "SweepableEstimatorGenerator": true diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj index df17513e14..ed8374d863 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Microsoft.ML.AutoML.SourceGenerator.csproj @@ -31,6 +31,11 @@ + + True + True + SweepableEstimatorFactory.tt + True True @@ -54,6 +59,10 @@ + + TextTemplatingFilePreprocessor + SweepableEstimatorFactory.cs + TextTemplatingFilePreprocessor EstimatorType.cs diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs new file mode 100644 index 0000000000..f8c5dba1f1 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SweepableEstimatorFactoryGenerator.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; +using Microsoft.ML.AutoML.SourceGenerator.Template; + +namespace Microsoft.ML.AutoML.SourceGenerator +{ + [Generator] + public class SweepableEstimatorFactoryGenerator : ISourceGenerator + { + private const string className = "SweepableEstimatorFactory"; + + public void Execute(GeneratorExecutionContext context) + { + if (context.AdditionalFiles.Where(f => f.Path.Contains("code_gen_flag.json")).First() is AdditionalText text) + { + var json = text.GetText().ToString(); + var flags = JsonSerializer.Deserialize>(json); + if (flags.TryGetValue(nameof(SweepableEstimatorFactoryGenerator), out var res) && res == false) + { + return; + } + } + + var trainers = context.AdditionalFiles.Where(f => f.Path.Contains("trainer-estimators.json")) + .SelectMany(file => Utils.GetEstimatorsFromJson(file.GetText().ToString()).Estimators, (text, estimator) => (estimator.FunctionName, estimator.EstimatorTypes, estimator.SearchOption)) + .SelectMany(union => union.EstimatorTypes.Select(t => (Utils.CreateEstimatorName(union.FunctionName, t), Utils.ToTitleCase(union.SearchOption)))) + .ToArray(); + + var transformers = context.AdditionalFiles.Where(f => f.Path.Contains("transformer-estimators.json")) + .SelectMany(file => Utils.GetEstimatorsFromJson(file.GetText().ToString()).Estimators, (text, estimator) => (estimator.FunctionName, estimator.EstimatorTypes, estimator.SearchOption)) + .SelectMany(union => union.EstimatorTypes.Select(t => (Utils.CreateEstimatorName(union.FunctionName, t), Utils.ToTitleCase(union.SearchOption)))) + .ToArray(); + + var code = new SweepableEstimatorFactory() + { + NameSpace = Constant.CodeGeneratorNameSpace, + EstimatorNames = trainers.Concat(transformers), + }; + + context.AddSource(className + ".cs", SourceText.From(code.TransformText(), Encoding.UTF8)); + } + + public void Initialize(GeneratorInitializationContext context) + { + } + } +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs new file mode 100644 index 0000000000..649537e3bf --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs @@ -0,0 +1,329 @@ +// ------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version: 17.0.0.0 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// ------------------------------------------------------------------------------ +namespace Microsoft.ML.AutoML.SourceGenerator.Template +{ + using System.Linq; + using System.Text; + using System.Collections.Generic; + using System; + + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class SweepableEstimatorFactory : SweepableEstimatorFactoryBase + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write("\r\nusing System;\r\nusing System.Collections.Generic;\r\nusing System.Text;\r\nusing New" + + "tonsoft.Json;\r\nusing Newtonsoft.Json.Linq;\r\nusing Microsoft.ML.SearchSpace;\r\nusi" + + "ng Microsoft.ML;\r\n\r\nnamespace "); + this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); + this.Write("\r\n{\r\n internal static class SweepableEsitmatorFactory\r\n {\r\n"); + foreach((var estimator, var tOption) in EstimatorNames){ + this.Write(" public static "); + this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); + this.Write(" Create"); + this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); + this.Write("("); + this.Write(this.ToStringHelper.ToStringWithCulture(tOption)); + this.Write(" defaultOption, SearchSpace<"); + this.Write(this.ToStringHelper.ToStringWithCulture(tOption)); + this.Write("> searchSpace = null)\r\n {\r\n if(searchSpace == null){\r\n " + + " searchSpace = new SearchSpace<"); + this.Write(this.ToStringHelper.ToStringWithCulture(tOption)); + this.Write(">(defaultOption);\r\n }\r\n\r\n return new "); + this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); + this.Write("(defaultOption, searchSpace);\r\n }\r\n\r\n"); +} + this.Write(" }\r\n}\r\n\r\n"); + return this.GenerationEnvironment.ToString(); + } + +public string NameSpace {get;set;} +public IEnumerable<(string, string)> EstimatorNames {get;set;} + + } + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class SweepableEstimatorFactoryBase + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + protected System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; + } + set + { + this.generationEnvironmentField = value; + } + } + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + } + return this.errorsField; + } + } + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); + } + return this.indentLengthsField; + } + } + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; + } + } + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; + } + set + { + this.sessionField = value; + } + } + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } + } + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; + } + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + } + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); + } + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); + } + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); + } + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; + } + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; + } + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; + } + set + { + if ((value != null)) + { + this.formatProviderField = value; + } + } + } + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { + throw new global::System.ArgumentNullException("objectToConvert"); + } + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); + } + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); + } + } + } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion + } + #endregion +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt new file mode 100644 index 0000000000..bbf6298711 --- /dev/null +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt @@ -0,0 +1,36 @@ +<#@ template language="C#" linePragmas="false" visibility = "internal" #> +<#@ assembly name="System.Core" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Text" #> +<#@ import namespace="System.Collections.Generic" #> + +using System; +using System.Collections.Generic; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Microsoft.ML.SearchSpace; +using Microsoft.ML; + +namespace <#=NameSpace#> +{ + internal static class SweepableEsitmatorFactory + { +<# foreach((var estimator, var tOption) in EstimatorNames){#> + public static <#=estimator#> Create<#=estimator#>(<#=tOption#> defaultOption, SearchSpace<<#=tOption#>> searchSpace = null) + { + if(searchSpace == null){ + searchSpace = new SearchSpace<<#=tOption#>>(defaultOption); + } + + return new <#=estimator#>(defaultOption, searchSpace); + } + +<#}#> + } +} + +<#+ +public string NameSpace {get;set;} +public IEnumerable<(string, string)> EstimatorNames {get;set;} +#> From 50d2c286a3d5f55e1fd84666772dabf5fb8876f6 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 11:16:09 -0700 Subject: [PATCH 04/16] add tests for sweepable estimator pipeline and multi-model pipeline --- .../SweepableEstimatorPipelineTest.cs | 90 ++++++++++++++++++- .../Template/SweepableEstimatorFactory.cs | 2 +- .../Template/SweepableEstimatorFactory.tt | 2 +- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 2b67e1d996..76195bdf7d 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -11,17 +11,24 @@ using ApprovalTests.Reporters; using FluentAssertions; using Microsoft.ML.TestFramework; -using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; +using Microsoft.ML.AutoML.CodeGen; +using System.Text.Json; namespace Microsoft.ML.AutoML.Test { public class SweepableEstimatorPipelineTest : BaseTestClass { + private readonly JsonSerializerOptions _jsonSerializerOptions; + public SweepableEstimatorPipelineTest(ITestOutputHelper output) : base(output) { + this._jsonSerializerOptions = new JsonSerializerOptions() + { + WriteIndented = true, + }; } [Fact] @@ -51,5 +58,86 @@ public void MultiModelPipeline_append_test() pipeline.BuildSweepableEstimatorPipeline("e0 * e2").ToString().Should().Be("Concatenate=>ApplyOnnxModel"); pipeline.BuildSweepableEstimatorPipeline("e1 * Nil").ToString().Should().Be("ConvertType"); } + + [Fact] + public void MultiModelPipeline_append_pipeline_test() + { + var e1 = new SweepableEstimator(CodeGen.EstimatorType.Concatenate); + var e2 = new SweepableEstimator(CodeGen.EstimatorType.ConvertType); + var e3 = new SweepableEstimator(CodeGen.EstimatorType.ApplyOnnxModel); + var e4 = new SweepableEstimator(CodeGen.EstimatorType.LightGbmBinary); + var e5 = new SweepableEstimator(CodeGen.EstimatorType.FastTreeBinary); + + var pipeline1 = new MultiModelPipeline(); + var pipeline2 = new MultiModelPipeline(); + + pipeline1 = pipeline1.Append(e1 + e2 * e3); + pipeline2 = pipeline2.Append(e1 * (e3 + e4) + e5); + + pipeline1 = pipeline1.Append(pipeline2); + + pipeline1.Schema.ToString().Should().Be("(e0 + e1 * e2) * (e3 * (e4 + e5) + e6)"); + } + + [Fact] + public void SweepableEstimatorPipeline_search_space_test() + { + var pipeline = this.CreateSweepbaleEstimatorPipeline(); + pipeline.SearchSpace.FeatureSpaceDim.Should().Be(15); + + // TODO + // verify other properties in search space. + } + + [Fact] + public void SweepableEstimatorPipeline_can_be_created_from_MultiModelPipeline() + { + var multiModelPipeline = this.CreateMultiModelPipeline(); + var pipelines = multiModelPipeline.PipelineIds; + + pipelines.Should().BeEquivalentTo("e0 * e3 * e4", "e1 * e2 * e3 * e4", "e0 * Nil * e4", "e1 * e2 * Nil * e4", "Nil * e3 * e4", "e0 * e3 * e5", "e1 * e2 * e3 * e5", "e0 * Nil * e5", "e1 * e2 * Nil * e5", "Nil * e3 * e5", "Nil * Nil * e4", "Nil * Nil * e5"); + var singleModelPipeline = multiModelPipeline.BuildSweepableEstimatorPipeline(pipelines[0]); + singleModelPipeline.ToString().Should().Be("ReplaceMissingValues=>Concatenate=>LightGbmBinary"); + singleModelPipeline = multiModelPipeline.BuildSweepableEstimatorPipeline(pipelines[2]); + singleModelPipeline.ToString().Should().Be("ReplaceMissingValues=>LightGbmBinary"); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void SweepableEstimatorPipeline_search_space_init_value_test() + { + var singleModelPipeline = this.CreateSweepbaleEstimatorPipeline(); + var defaultParam = singleModelPipeline.SearchSpace.SampleFromFeatureSpace(singleModelPipeline.SearchSpace.Default); + Approvals.Verify(JsonSerializer.Serialize(defaultParam, this._jsonSerializerOptions)); + } + + private SweepableEstimatorPipeline CreateSweepbaleEstimatorPipeline() + { + var concat = SweepableEstimatorFactory.CreateConcatenate(new ConcatOption()); + var replaceMissingValue = SweepableEstimatorFactory.CreateReplaceMissingValues(new ReplaceMissingValueOption()); + var oneHot = SweepableEstimatorFactory.CreateOneHotEncoding(new OneHotOption()); + var lightGbm = SweepableEstimatorFactory.CreateLightGbmBinary(new LgbmOption()); + var fastTree = SweepableEstimatorFactory.CreateFastTreeBinary(new FastTreeOption()); + + var pipeline = new SweepableEstimatorPipeline(new SweepableEstimator[] { concat, replaceMissingValue, oneHot, lightGbm, fastTree }); + return pipeline; + } + + private MultiModelPipeline CreateMultiModelPipeline() + { + var concat = SweepableEstimatorFactory.CreateConcatenate(new ConcatOption()); + var replaceMissingValue = SweepableEstimatorFactory.CreateReplaceMissingValues(new ReplaceMissingValueOption()); + var oneHot = SweepableEstimatorFactory.CreateOneHotEncoding(new OneHotOption()); + var lightGbm = SweepableEstimatorFactory.CreateLightGbmBinary(new LgbmOption()); + var fastTree = SweepableEstimatorFactory.CreateFastTreeBinary(new FastTreeOption()); + + var pipeline = new MultiModelPipeline(); + pipeline = pipeline.AppendOrSkip(replaceMissingValue + replaceMissingValue * oneHot); + pipeline = pipeline.AppendOrSkip(concat); + pipeline = pipeline.Append(lightGbm + fastTree); + + return pipeline; + } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs index 649537e3bf..820d931f14 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.cs @@ -29,7 +29,7 @@ public virtual string TransformText() "tonsoft.Json;\r\nusing Newtonsoft.Json.Linq;\r\nusing Microsoft.ML.SearchSpace;\r\nusi" + "ng Microsoft.ML;\r\n\r\nnamespace "); this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); - this.Write("\r\n{\r\n internal static class SweepableEsitmatorFactory\r\n {\r\n"); + this.Write("\r\n{\r\n internal static class SweepableEstimatorFactory\r\n {\r\n"); foreach((var estimator, var tOption) in EstimatorNames){ this.Write(" public static "); this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt index bbf6298711..c9fdee1607 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SweepableEstimatorFactory.tt @@ -14,7 +14,7 @@ using Microsoft.ML; namespace <#=NameSpace#> { - internal static class SweepableEsitmatorFactory + internal static class SweepableEstimatorFactory { <# foreach((var estimator, var tOption) in EstimatorNames){#> public static <#=estimator#> Create<#=estimator#>(<#=tOption#> defaultOption, SearchSpace<<#=tOption#>> searchSpace = null) From 8682502043ef6ad2db1e001b687e673028f2bab8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 14:15:00 -0700 Subject: [PATCH 05/16] add sweepable extension tests and more tests --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 169 ++++++++++++++++++ .../API/SweepableExtension.cs | 30 +++- .../Converter/MultiModelPipelineConverter.cs | 34 ++++ .../Converter/SweepableEstimatorConverter.cs | 37 ++++ .../SweepableEstimatorPipelineConverter.cs | 39 ++++ .../SweepableEstimator/Estimator.cs | 1 + .../SweepableEstimator/MultiModelPipeline.cs | 2 + .../SweepableEstimator/SweepableEstimator.cs | 2 + .../SweepableEstimatorPipeline.cs | 2 + ...delPipeline_search_space_test.approved.txt | 0 ...delPipeline_search_space_test.received.txt | 61 +++++++ ..._search_space_init_value_test.approved.txt | 28 +++ ...EstimatorAndBinaryClassifiers.approved.txt | 66 +++++++ ...IEstimatorAndMultiClassifiers.approved.txt | 84 +++++++++ ...neFromIEstimatorAndRegressors.approved.txt | 84 +++++++++ ...eEstimatorAndMultiClassifiers.approved.txt | 90 ++++++++++ ...eEstimatorAndMultiClassifiers.received.txt | 90 ++++++++++ ...orPipelineAndMultiClassifiers.approved.txt | 88 +++++++++ ...orPipelineAndMultiClassifiers.received.txt | 88 +++++++++ .../SweepableExtensionTest.cs | 147 +++++++++++++++ .../Template/EstimatorType.cs | 6 +- .../Template/EstimatorType.tt | 1 + 22 files changed, 1145 insertions(+), 4 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs create mode 100644 src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt create mode 100644 test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index b704188f68..3b822b647f 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -3,6 +3,10 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.Contracts; +using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.Data; using Microsoft.ML.SearchSpace; @@ -289,5 +293,170 @@ internal SweepableEstimator CreateSweepableEstimator(Func factory(context, param.AsType()), ss); } + + internal SweepableEstimator[] BinaryClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, + SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) + { + var res = new List(); + + if (useFastTree) + { + fastTreeOption = fastTreeOption ?? new FastTreeOption(); + fastTreeOption.LabelColumnName = labelColumnName; + fastTreeOption.FeatureColumnName = featureColumnName; + fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastTreeBinary(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + } + + if (useFastForest) + { + fastForestOption = fastForestOption ?? new FastForestOption(); + fastForestOption.LabelColumnName = labelColumnName; + fastForestOption.FeatureColumnName = featureColumnName; + fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastForestBinary(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + } + + if (useLgbm) + { + lgbmOption = lgbmOption ?? new LgbmOption(); + lgbmOption.LabelColumnName = labelColumnName; + lgbmOption.FeatureColumnName = featureColumnName; + lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLightGbmBinary(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + } + + if (useLbfgs) + { + lbfgsOption = lbfgsOption ?? new LbfgsOption(); + lbfgsOption.LabelColumnName = labelColumnName; + lbfgsOption.FeatureColumnName = featureColumnName; + lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionBinary(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + } + + if (useSdca) + { + sdcaOption = sdcaOption ?? new SdcaOption(); + sdcaOption.LabelColumnName = labelColumnName; + sdcaOption.FeatureColumnName = featureColumnName; + sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionBinary(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + } + + return res.ToArray(); + } + + internal SweepableEstimator[] MultiClassification(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, + SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) + { + var res = new List(); + + if (useFastTree) + { + fastTreeOption = fastTreeOption ?? new FastTreeOption(); + fastTreeOption.LabelColumnName = labelColumnName; + fastTreeOption.FeatureColumnName = featureColumnName; + fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastTreeOva(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + } + + if (useFastForest) + { + fastForestOption = fastForestOption ?? new FastForestOption(); + fastForestOption.LabelColumnName = labelColumnName; + fastForestOption.FeatureColumnName = featureColumnName; + fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastForestOva(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + } + + if (useLgbm) + { + lgbmOption = lgbmOption ?? new LgbmOption(); + lgbmOption.LabelColumnName = labelColumnName; + lgbmOption.FeatureColumnName = featureColumnName; + lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLightGbmMulti(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + } + + if (useLbfgs) + { + lbfgsOption = lbfgsOption ?? new LbfgsOption(); + lbfgsOption.LabelColumnName = labelColumnName; + lbfgsOption.FeatureColumnName = featureColumnName; + lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLbfgsLogisticRegressionOva(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateLbfgsMaximumEntropyMulti(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + } + + if (useSdca) + { + sdcaOption = sdcaOption ?? new SdcaOption(); + sdcaOption.LabelColumnName = labelColumnName; + sdcaOption.FeatureColumnName = featureColumnName; + sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateSdcaMaximumEntropyMulti(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateSdcaLogisticRegressionOva(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + } + + return res.ToArray(); + } + + internal SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, bool useFastForest = true, bool useLgbm = true, bool useFastTree = true, bool useLbfgs = true, bool useSdca = true, + FastTreeOption fastTreeOption = null, LgbmOption lgbmOption = null, FastForestOption fastForestOption = null, LbfgsOption lbfgsOption = null, SdcaOption sdcaOption = null, + SearchSpace fastTreeSearchSpace = null, SearchSpace lgbmSearchSpace = null, SearchSpace fastForestSearchSpace = null, SearchSpace lbfgsSearchSpace = null, SearchSpace sdcaSearchSpace = null) + { + var res = new List(); + + if (useFastTree) + { + fastTreeOption = fastTreeOption ?? new FastTreeOption(); + fastTreeOption.LabelColumnName = labelColumnName; + fastTreeOption.FeatureColumnName = featureColumnName; + fastTreeOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastTreeRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + res.Add(SweepableEstimatorFactory.CreateFastTreeTweedieRegression(fastTreeOption, fastTreeSearchSpace ?? new SearchSpace())); + } + + if (useFastForest) + { + fastForestOption = fastForestOption ?? new FastForestOption(); + fastForestOption.LabelColumnName = labelColumnName; + fastForestOption.FeatureColumnName = featureColumnName; + fastForestOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateFastForestRegression(fastForestOption, fastForestSearchSpace ?? new SearchSpace())); + } + + if (useLgbm) + { + lgbmOption = lgbmOption ?? new LgbmOption(); + lgbmOption.LabelColumnName = labelColumnName; + lgbmOption.FeatureColumnName = featureColumnName; + lgbmOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLightGbmRegression(lgbmOption, lgbmSearchSpace ?? new SearchSpace())); + } + + if (useLbfgs) + { + lbfgsOption = lbfgsOption ?? new LbfgsOption(); + lbfgsOption.LabelColumnName = labelColumnName; + lbfgsOption.FeatureColumnName = featureColumnName; + lbfgsOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateLbfgsPoissonRegressionRegression(lbfgsOption, lbfgsSearchSpace ?? new SearchSpace())); + } + + if (useSdca) + { + sdcaOption = sdcaOption ?? new SdcaOption(); + sdcaOption.LabelColumnName = labelColumnName; + sdcaOption.FeatureColumnName = featureColumnName; + sdcaOption.ExampleWeightColumnName = exampleWeightColumnName; + res.Add(SweepableEstimatorFactory.CreateSdcaRegression(sdcaOption, sdcaSearchSpace ?? new SearchSpace())); + } + + return res.ToArray(); + } } } diff --git a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs index baf72267ce..1848d8c128 100644 --- a/src/Microsoft.ML.AutoML/API/SweepableExtension.cs +++ b/src/Microsoft.ML.AutoML/API/SweepableExtension.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; namespace Microsoft.ML.AutoML @@ -27,7 +28,34 @@ public static SweepableEstimatorPipeline Append(this SweepableEstimator estimato public static SweepableEstimatorPipeline Append(this SweepableEstimator estimator, IEstimator estimator1) { - return estimator.Append(estimator1); + return new SweepableEstimatorPipeline().Append(estimator).Append(estimator1); + } + + public static MultiModelPipeline Append(this IEstimator estimator, params SweepableEstimator[] estimators) + { + var sweepableEstimator = new SweepableEstimator((context, parameter) => estimator, new SearchSpace.SearchSpace()); + var multiModelPipeline = new MultiModelPipeline().Append(sweepableEstimator).Append(estimators); + + return multiModelPipeline; + } + + public static MultiModelPipeline Append(this SweepableEstimatorPipeline pipeline, params SweepableEstimator[] estimators) + { + var multiModelPipeline = new MultiModelPipeline(); + foreach (var estimator in pipeline.Estimators) + { + multiModelPipeline = multiModelPipeline.Append(estimator); + } + + return multiModelPipeline.Append(estimators); + } + + public static MultiModelPipeline Append(this SweepableEstimator estimator, params SweepableEstimator[] estimators) + { + var multiModelPipeline = new MultiModelPipeline(); + multiModelPipeline = multiModelPipeline.Append(estimator); + + return multiModelPipeline.Append(estimators); } } } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs new file mode 100644 index 0000000000..d3107c939a --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/MultiModelPipelineConverter.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Microsoft.ML.AutoML +{ + internal class MultiModelPipelineConverter : JsonConverter + { + public override MultiModelPipeline Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jValue = JsonValue.Parse(ref reader); + var schema = jValue["schema"].GetValue(); + var estimators = jValue["estimator"].GetValue>(); + + return new MultiModelPipeline(estimators, Entity.FromExpression(schema)); + } + + public override void Write(Utf8JsonWriter writer, MultiModelPipeline value, JsonSerializerOptions options) + { + var jsonObject = JsonNode.Parse("{}"); + jsonObject["schema"] = value.Schema.ToString(); + jsonObject["estimators"] = JsonValue.Create(value.Estimators); + + jsonObject.WriteTo(writer, options); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs new file mode 100644 index 0000000000..362de6eaa4 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorConverter.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.ML.AutoML.CodeGen; +using Microsoft.ML.SearchSpace; + +namespace Microsoft.ML.AutoML +{ + internal class SweepableEstimatorConverter : JsonConverter + { + public override SweepableEstimator Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jsonObject = JsonValue.Parse(ref reader); + var estimatorType = jsonObject["estimatorType"].GetValue(); + var parameter = jsonObject["parameter"].GetValue(); + var estimator = new SweepableEstimator(estimatorType); + estimator.Parameter = parameter; + + return estimator; + } + + public override void Write(Utf8JsonWriter writer, SweepableEstimator value, JsonSerializerOptions options) + { + var jObject = JsonObject.Parse("{}"); + jObject["estimatorType"] = JsonValue.Create(value.EstimatorType); + jObject["parameter"] = JsonValue.Create(value.Parameter); + jObject.WriteTo(writer, options); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs new file mode 100644 index 0000000000..403654ea21 --- /dev/null +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Converter/SweepableEstimatorPipelineConverter.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.ML.SearchSpace; +using Newtonsoft.Json; + +namespace Microsoft.ML.AutoML +{ + internal class SweepableEstimatorPipelineConverter : JsonConverter + { + public override SweepableEstimatorPipeline Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jNode = JsonNode.Parse(ref reader); + var parameter = jNode["parameter"].GetValue(); + var estimators = jNode["estimators"].GetValue(); + var pipeline = new SweepableEstimatorPipeline(estimators, parameter); + + return pipeline; + } + + public override void Write(Utf8JsonWriter writer, SweepableEstimatorPipeline value, JsonSerializerOptions options) + { + var parameter = value.Parameter; + var estimators = value.Estimators; + var jNode = JsonNode.Parse("{}"); + jNode["parameter"] = JsonValue.Create(parameter); + jNode["estimators"] = JsonValue.Create(estimators); + + jNode.WriteTo(writer, options); + } + } +} diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs index d26fa97134..11339ede14 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimator.cs @@ -12,6 +12,7 @@ internal class Estimator protected Estimator() { this.Parameter = Parameter.CreateNestedParameter(); + this.EstimatorType = EstimatorType.Unknown; } internal Estimator(EstimatorType estimatorType) diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs index 55777bc9ba..3be0a7dff8 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/MultiModelPipeline.cs @@ -6,9 +6,11 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Text.Json.Serialization; namespace Microsoft.ML.AutoML { + [JsonConverter(typeof(MultiModelPipelineConverter))] internal class MultiModelPipeline { private static readonly StringEntity _nilStringEntity = new StringEntity("Nil"); diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs index d1e65bf898..3dc3186978 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimator.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Text.Json.Serialization; using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.SearchSpace; @@ -12,6 +13,7 @@ namespace Microsoft.ML.AutoML /// /// Estimator with search space. /// + [JsonConverter(typeof(SweepableEstimatorConverter))] internal class SweepableEstimator : Estimator { private readonly Func> _factory; diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs index 5aae62e2d1..1f1909638d 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/SweepableEstimatorPipeline.cs @@ -5,11 +5,13 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Serialization; using Microsoft.ML.Data; using Microsoft.ML.SearchSpace; namespace Microsoft.ML.AutoML { + [JsonConverter(typeof(SweepableEstimatorPipelineConverter))] internal class SweepableEstimatorPipeline { private readonly List _estimators; diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt new file mode 100644 index 0000000000..7d3e8e1861 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.received.txt @@ -0,0 +1,61 @@ +{ + "0": { + "FeatureSpaceDim": 0, + "Default": [], + "Step": [] + }, + "1": { + "FeatureSpaceDim": 0, + "Default": [], + "Step": [] + }, + "2": { + "FeatureSpaceDim": 0, + "Default": [], + "Step": [] + }, + "3": { + "FeatureSpaceDim": 9, + "Default": [ + 1, + 0, + 1, + 1, + 0.71428571428571408, + 0, + 0, + 0, + 1 + ], + "Step": [ + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "4": { + "FeatureSpaceDim": 6, + "Default": [ + 1, + 0.89689626841273451, + 0.71428571428571408, + 0.55365468248122718, + 0, + 0 + ], + "Step": [ + null, + null, + null, + null, + null, + null + ] + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt new file mode 100644 index 0000000000..e4b6191616 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt @@ -0,0 +1,28 @@ +{ + "0": {}, + "1": {}, + "2": {}, + "3": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 255, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + }, + "4": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 255, + "FeatureFraction": 1, + "LearningRate": 0.099999999999999978, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt new file mode 100644 index 0000000000..fa86919915 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt @@ -0,0 +1,66 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeBinary", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmBinary", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionBinary", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "SdcaLogisticRegressionBinary", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..00a3d413c7 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,84 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt new file mode 100644 index 0000000000..00a3d413c7 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt @@ -0,0 +1,84 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..d6f4d7d027 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt @@ -0,0 +1,90 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt new file mode 100644 index 0000000000..ada341d69d --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt @@ -0,0 +1,90 @@ +{ + "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", + "estimators": { + "e0": { + "estimatorType": "FastForestBinary", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Feature" + } + }, + "e1": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e2": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt new file mode 100644 index 0000000000..21058f9cbd --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt @@ -0,0 +1,88 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt new file mode 100644 index 0000000000..adcaa9ec62 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt @@ -0,0 +1,88 @@ +{ + "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", + "estimators": { + "e0": { + "estimatorType": "Unknown", + "parameter": {} + }, + "e1": { + "estimatorType": "FeaturizeText", + "parameter": {} + }, + "e2": { + "estimatorType": "FastTreeOva", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "NumberOfTrees": 4, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "LearningRate": 0.10000000000000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e3": { + "estimatorType": "FastForestOva", + "parameter": { + "NumberOfTrees": 4, + "NumberOfLeaves": 4, + "FeatureFraction": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e4": { + "estimatorType": "LightGbmMulti", + "parameter": { + "NumberOfLeaves": 4, + "MinimumExampleCountPerLeaf": 20, + "LearningRate": 1, + "NumberOfTrees": 4, + "SubsampleFraction": 1, + "MaximumBinCountPerFeature": 256, + "FeatureFraction": 1, + "L1Regularization": 2.0000000000000001E-10, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e5": { + "estimatorType": "LbfgsLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e6": { + "estimatorType": "LbfgsMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 1, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e7": { + "estimatorType": "SdcaMaximumEntropyMulti", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + }, + "e8": { + "estimatorType": "SdcaLogisticRegressionOva", + "parameter": { + "L1Regularization": 1, + "L2Regularization": 0.100000001, + "LabelColumnName": "Label", + "FeatureColumnName": "Features" + } + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs new file mode 100644 index 0000000000..72df8a0dec --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -0,0 +1,147 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; +using Microsoft.ML.AutoML.CodeGen; +using FluentAssertions; +using Microsoft.ML.TestFramework; +using Xunit.Abstractions; +using ApprovalTests.Namers; +using ApprovalTests.Reporters; +using System.Text.Json.Serialization; +using System.Text.Json; +using ApprovalTests; + +namespace Microsoft.ML.AutoML.Test +{ + public class SweepableExtensionTest : BaseTestClass + { + private readonly JsonSerializerOptions _jsonSerializerOptions; + + public SweepableExtensionTest(ITestOutputHelper output) + : base(output) + { + this._jsonSerializerOptions = new JsonSerializerOptions() + { + WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(), + }, + }; + + this._jsonSerializerOptions.Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping; + } + + [Fact] + public void CreateSweepableEstimatorPipelineFromIEstimatorTest() + { + var context = new MLContext(); + var estimator = context.Transforms.Concatenate("output", "input"); + var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + + pipeline.ToString().Should().Be("Unknown=>FastForestBinary"); + } + + [Fact] + public void AppendIEstimatorToSweepabeEstimatorPipelineTest() + { + var context = new MLContext(); + var estimator = context.Transforms.Concatenate("output", "input"); + var pipeline = estimator.Append(SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption())); + pipeline = pipeline.Append(context.Transforms.CopyColumns("output", "input")); + + pipeline.ToString().Should().Be("Unknown=>FastForestBinary=>Unknown"); + } + + [Fact] + public void CreateSweepableEstimatorPipelineFromSweepableEstimatorTest() + { + var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); + var pipeline = estimator.Append(estimator); + + pipeline.ToString().Should().Be("FastForestBinary=>FastForestBinary"); + } + + [Fact] + public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorTest() + { + var context = new MLContext(); + var estimator = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()); + var pipeline = estimator.Append(context.Transforms.Concatenate("output", "input")); + + pipeline.ToString().Should().Be("FastForestBinary=>Unknown"); + + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().BinaryClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromIEstimatorAndRegressors() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() + { + var context = new MLContext(); + var pipeline = SweepableEstimatorFactory.CreateFastForestBinary(new FastForestOption()) + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + + [Fact] + [UseApprovalSubdirectory("ApprovalTests")] + [UseReporter(typeof(DiffReporter))] + public void CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers() + { + var context = new MLContext(); + var pipeline = context.Transforms.Concatenate("output", "input") + .Append(SweepableEstimatorFactory.CreateFeaturizeText(new FeaturizeTextOption())) + .Append(context.Auto().MultiClassification()); + + var json = JsonSerializer.Serialize(pipeline, this._jsonSerializerOptions); + Approvals.Verify(json); + } + } +} diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs index 7e1c51100a..f0f9f3a960 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.cs @@ -40,9 +40,9 @@ public virtual string TransformText() this.Write(this.ToStringHelper.ToStringWithCulture(e)); this.Write(",\r\n"); } - this.Write(" }\r\n\r\n public static class EstimatorTypeExtension\r\n {\r\n public st" + - "atic bool IsTrainer(this EstimatorType estimatorType)\r\n {\r\n sw" + - "itch(estimatorType)\r\n {\r\n"); + this.Write(" Unknown,\r\n }\r\n\r\n public static class EstimatorTypeExtension\r\n {\r" + + "\n public static bool IsTrainer(this EstimatorType estimatorType)\r\n " + + " {\r\n switch(estimatorType)\r\n {\r\n"); foreach(var estimator in TrainerNames){ this.Write(" case EstimatorType."); this.Write(this.ToStringHelper.ToStringWithCulture(estimator)); diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt index d95e7a3066..0fcb75f298 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/EstimatorType.tt @@ -18,6 +18,7 @@ namespace <#=NameSpace#> <# foreach(var e in TransformerNames){#> <#=e#>, <#}#> + Unknown, } public static class EstimatorTypeExtension From 58e4b80f12619fa38d2d4b684b2920502666ac03 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 14 Mar 2022 15:08:34 -0700 Subject: [PATCH 06/16] fix tests --- .../SweepableEstimatorPipelineTest.cs | 5 +++++ test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 76195bdf7d..ec97716693 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -29,6 +29,11 @@ public SweepableEstimatorPipelineTest(ITestOutputHelper output) { WriteIndented = true, }; + + if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) + { + Approvals.UseAssemblyLocationForApprovedFiles(); + } } [Fact] diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index 72df8a0dec..b74b9e9730 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -35,6 +35,11 @@ public SweepableExtensionTest(ITestOutputHelper output) }; this._jsonSerializerOptions.Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping; + + if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) + { + Approvals.UseAssemblyLocationForApprovedFiles(); + } } [Fact] From db85b41dbe0d3b2676f8ac918953d965f0bc169b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 11:12:48 -0700 Subject: [PATCH 07/16] fix tests --- .../Microsoft.ML.AutoML.Tests.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj index ea56f75d8b..076b1aa229 100644 --- a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj +++ b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj @@ -15,8 +15,8 @@ - - PreserveNewest + + Always PreserveNewest From 4cdb321d281e96e01ffec5b5ca42133da48721c4 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 12:07:23 -0700 Subject: [PATCH 08/16] fix build --- src/Microsoft.ML.SearchSpace/Parameter.cs | 2 +- ...eEstimatorAndMultiClassifiers.received.txt | 90 ------------------- ...orPipelineAndMultiClassifiers.received.txt | 88 ------------------ 3 files changed, 1 insertion(+), 179 deletions(-) delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt diff --git a/src/Microsoft.ML.SearchSpace/Parameter.cs b/src/Microsoft.ML.SearchSpace/Parameter.cs index a584ef10e6..495e535809 100644 --- a/src/Microsoft.ML.SearchSpace/Parameter.cs +++ b/src/Microsoft.ML.SearchSpace/Parameter.cs @@ -50,7 +50,7 @@ public enum ParameterType } /// - /// is used to save sweeping result from and is used to restore mlnet pipeline from sweepable pipline. + /// is used to save sweeping result from tuner and is used to restore mlnet pipeline from sweepable pipline. /// [JsonConverter(typeof(ParameterConverter))] public sealed class Parameter : IDictionary diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt deleted file mode 100644 index ada341d69d..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.received.txt +++ /dev/null @@ -1,90 +0,0 @@ -{ - "schema": "e0 * (e1 + e2 + e3 + e4 + e5 + e6 + e7)", - "estimators": { - "e0": { - "estimatorType": "FastForestBinary", - "parameter": { - "NumberOfTrees": 4, - "NumberOfLeaves": 4, - "FeatureFraction": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Feature" - } - }, - "e1": { - "estimatorType": "FastTreeOva", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "NumberOfTrees": 4, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e2": { - "estimatorType": "FastForestOva", - "parameter": { - "NumberOfTrees": 4, - "NumberOfLeaves": 4, - "FeatureFraction": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e3": { - "estimatorType": "LightGbmMulti", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "LearningRate": 1, - "NumberOfTrees": 4, - "SubsampleFraction": 1, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e4": { - "estimatorType": "LbfgsLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e5": { - "estimatorType": "LbfgsMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e6": { - "estimatorType": "SdcaMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e7": { - "estimatorType": "SdcaLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - } - } -} \ No newline at end of file diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt deleted file mode 100644 index adcaa9ec62..0000000000 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.received.txt +++ /dev/null @@ -1,88 +0,0 @@ -{ - "schema": "e0 * e1 * (e2 + e3 + e4 + e5 + e6 + e7 + e8)", - "estimators": { - "e0": { - "estimatorType": "Unknown", - "parameter": {} - }, - "e1": { - "estimatorType": "FeaturizeText", - "parameter": {} - }, - "e2": { - "estimatorType": "FastTreeOva", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "NumberOfTrees": 4, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e3": { - "estimatorType": "FastForestOva", - "parameter": { - "NumberOfTrees": 4, - "NumberOfLeaves": 4, - "FeatureFraction": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e4": { - "estimatorType": "LightGbmMulti", - "parameter": { - "NumberOfLeaves": 4, - "MinimumExampleCountPerLeaf": 20, - "LearningRate": 1, - "NumberOfTrees": 4, - "SubsampleFraction": 1, - "MaximumBinCountPerFeature": 256, - "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e5": { - "estimatorType": "LbfgsLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e6": { - "estimatorType": "LbfgsMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 1, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e7": { - "estimatorType": "SdcaMaximumEntropyMulti", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - }, - "e8": { - "estimatorType": "SdcaLogisticRegressionOva", - "parameter": { - "L1Regularization": 1, - "L2Regularization": 0.100000001, - "LabelColumnName": "Label", - "FeatureColumnName": "Features" - } - } - } -} \ No newline at end of file From 8fe74506927f313115ef7209cdf8bffc40f40a52 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 13:13:23 -0700 Subject: [PATCH 09/16] fix tests --- .../SweepableEstimatorPipelineTest.cs | 3 +-- test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index ec97716693..6662d85a49 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -18,12 +18,11 @@ namespace Microsoft.ML.AutoML.Test { - public class SweepableEstimatorPipelineTest : BaseTestClass + public class SweepableEstimatorPipelineTest { private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableEstimatorPipelineTest(ITestOutputHelper output) - : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index b74b9e9730..91ea6cebee 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -18,12 +18,13 @@ namespace Microsoft.ML.AutoML.Test { - public class SweepableExtensionTest : BaseTestClass +#pragma warning disable MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass + public class SweepableExtensionTest +#pragma warning restore MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass { private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableExtensionTest(ITestOutputHelper output) - : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { From 904da0c6590f3c12e28a36bb3edaaf254673c121 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 13:26:08 -0700 Subject: [PATCH 10/16] update --- .../Microsoft.ML.AutoML.Tests.csproj | 7 ++++--- .../SweepableEstimatorPipelineTest.cs | 8 +++----- .../SweepableExtensionTest.cs | 10 +--------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj index 076b1aa229..ee452addc5 100644 --- a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj +++ b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj @@ -15,9 +15,10 @@ - - Always - + + PreserveNewest + %(Filename).txt + PreserveNewest diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 6662d85a49..5f2b9b568a 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -18,7 +18,9 @@ namespace Microsoft.ML.AutoML.Test { +#pragma warning disable MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass public class SweepableEstimatorPipelineTest +#pragma warning restore MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass { private readonly JsonSerializerOptions _jsonSerializerOptions; @@ -29,10 +31,7 @@ public SweepableEstimatorPipelineTest(ITestOutputHelper output) WriteIndented = true, }; - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } + Approvals.UseAssemblyLocationForApprovedFiles(); } [Fact] @@ -107,7 +106,6 @@ public void SweepableEstimatorPipeline_can_be_created_from_MultiModelPipeline() } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void SweepableEstimatorPipeline_search_space_init_value_test() { diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index 91ea6cebee..9c91b6c170 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -37,10 +37,7 @@ public SweepableExtensionTest(ITestOutputHelper output) this._jsonSerializerOptions.Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping; - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } + Approvals.UseAssemblyLocationForApprovedFiles(); } [Fact] @@ -85,7 +82,6 @@ public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorT } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() { @@ -98,7 +94,6 @@ public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() { @@ -111,7 +106,6 @@ public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromIEstimatorAndRegressors() { @@ -124,7 +118,6 @@ public void CreateMultiModelPipelineFromIEstimatorAndRegressors() } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() { @@ -137,7 +130,6 @@ public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() } [Fact] - [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers() { From 2b055af8390c5165f0060b1b9a2f611a7fc677b5 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 13:26:58 -0700 Subject: [PATCH 11/16] fix warning --- .../SweepableEstimatorPipelineTest.cs | 4 +--- test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 5f2b9b568a..54ea8515cd 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -18,9 +18,7 @@ namespace Microsoft.ML.AutoML.Test { -#pragma warning disable MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass - public class SweepableEstimatorPipelineTest -#pragma warning restore MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass + public class SweepableEstimatorPipelineTest : BaseTestClass { private readonly JsonSerializerOptions _jsonSerializerOptions; diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index 9c91b6c170..f7adb4632e 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -18,9 +18,7 @@ namespace Microsoft.ML.AutoML.Test { -#pragma warning disable MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass - public class SweepableExtensionTest -#pragma warning restore MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass + public class SweepableExtensionTest : BaseTestClass { private readonly JsonSerializerOptions _jsonSerializerOptions; From 9a59187c385b3c83211e28b2e6dcb1a8ed151b54 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 13:27:33 -0700 Subject: [PATCH 12/16] fix build error --- test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs | 1 + test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 54ea8515cd..f12bc50383 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -23,6 +23,7 @@ public class SweepableEstimatorPipelineTest : BaseTestClass private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableEstimatorPipelineTest(ITestOutputHelper output) + : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index f7adb4632e..f7ad8f98a2 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -23,6 +23,7 @@ public class SweepableExtensionTest : BaseTestClass private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableExtensionTest(ITestOutputHelper output) + : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { From 9aa4af7aae7eb01c233344b15b9d76a2215c2e9e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 14:09:47 -0700 Subject: [PATCH 13/16] fix tests --- .../SweepableEstimatorPipelineTest.cs | 3 +-- test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index f12bc50383..730b61a208 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -18,12 +18,11 @@ namespace Microsoft.ML.AutoML.Test { - public class SweepableEstimatorPipelineTest : BaseTestClass + public class SweepableEstimatorPipelineTest { private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableEstimatorPipelineTest(ITestOutputHelper output) - : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index f7ad8f98a2..9c91b6c170 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -18,12 +18,13 @@ namespace Microsoft.ML.AutoML.Test { - public class SweepableExtensionTest : BaseTestClass +#pragma warning disable MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass + public class SweepableExtensionTest +#pragma warning restore MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass { private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableExtensionTest(ITestOutputHelper output) - : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { From 77ab49fb96a554689139b6ef86fe9d46ec97b5b4 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 22 Mar 2022 17:31:27 -0700 Subject: [PATCH 14/16] fix tests --- .../Microsoft.ML.AutoML.Tests.csproj | 5 ++--- .../SweepableEstimatorPipelineTest.cs | 9 +++++++-- .../SweepableExtensionTest.cs | 15 +++++++++++---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj index ee452addc5..ea56f75d8b 100644 --- a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj +++ b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj @@ -15,10 +15,9 @@ - + PreserveNewest - %(Filename).txt - + PreserveNewest diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 730b61a208..07e67ca3e6 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -18,18 +18,22 @@ namespace Microsoft.ML.AutoML.Test { - public class SweepableEstimatorPipelineTest + public class SweepableEstimatorPipelineTest : BaseTestClass { private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableEstimatorPipelineTest(ITestOutputHelper output) + : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { WriteIndented = true, }; - Approvals.UseAssemblyLocationForApprovedFiles(); + if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) + { + Approvals.UseAssemblyLocationForApprovedFiles(); + } } [Fact] @@ -105,6 +109,7 @@ public void SweepableEstimatorPipeline_can_be_created_from_MultiModelPipeline() [Fact] [UseReporter(typeof(DiffReporter))] + [UseApprovalSubdirectory("ApprovalTests")] public void SweepableEstimatorPipeline_search_space_init_value_test() { var singleModelPipeline = this.CreateSweepbaleEstimatorPipeline(); diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index 9c91b6c170..e948d08696 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -18,13 +18,12 @@ namespace Microsoft.ML.AutoML.Test { -#pragma warning disable MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass - public class SweepableExtensionTest -#pragma warning restore MSML_ExtendBaseTestClass // Test classes should be derived from BaseTestClass or FunctionalTestBaseClass + public class SweepableExtensionTest : BaseTestClass { private readonly JsonSerializerOptions _jsonSerializerOptions; public SweepableExtensionTest(ITestOutputHelper output) + : base(output) { this._jsonSerializerOptions = new JsonSerializerOptions() { @@ -37,7 +36,10 @@ public SweepableExtensionTest(ITestOutputHelper output) this._jsonSerializerOptions.Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping; - Approvals.UseAssemblyLocationForApprovedFiles(); + if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) + { + Approvals.UseAssemblyLocationForApprovedFiles(); + } } [Fact] @@ -83,6 +85,7 @@ public void CreateSweepableEstimatorPipelineFromSweepableEstimatorAndIEstimatorT [Fact] [UseReporter(typeof(DiffReporter))] + [UseApprovalSubdirectory("ApprovalTests")] public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() { var context = new MLContext(); @@ -94,6 +97,7 @@ public void CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers() } [Fact] + [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() { @@ -106,6 +110,7 @@ public void CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers() } [Fact] + [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromIEstimatorAndRegressors() { @@ -118,6 +123,7 @@ public void CreateMultiModelPipelineFromIEstimatorAndRegressors() } [Fact] + [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() { @@ -130,6 +136,7 @@ public void CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers() } [Fact] + [UseApprovalSubdirectory("ApprovalTests")] [UseReporter(typeof(DiffReporter))] public void CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers() { From 909150f42c73a048fd3b4ce7362f50362f032107 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 23 Mar 2022 11:01:24 -0700 Subject: [PATCH 15/16] convert double/float to decimal when serializing to json in tests --- ..._search_space_init_value_test.approved.txt | 4 ++-- ...EstimatorAndBinaryClassifiers.approved.txt | 6 ++--- ...IEstimatorAndMultiClassifiers.approved.txt | 8 +++---- ...neFromIEstimatorAndRegressors.approved.txt | 8 +++---- ...eEstimatorAndMultiClassifiers.approved.txt | 8 +++---- ...orPipelineAndMultiClassifiers.approved.txt | 8 +++---- .../SweepableEstimatorPipelineTest.cs | 5 ++++ .../SweepableExtensionTest.cs | 2 +- .../Utils/DoubleToDecimalConverter.cs | 23 +++++++++++++++++++ .../Utils/FloatToDecimalConverter.cs | 23 +++++++++++++++++++ 10 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 test/Microsoft.ML.AutoML.Tests/Utils/DoubleToDecimalConverter.cs create mode 100644 test/Microsoft.ML.AutoML.Tests/Utils/FloatToDecimalConverter.cs diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt index e4b6191616..6ff945b59f 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt @@ -10,7 +10,7 @@ "SubsampleFraction": 1, "MaximumBinCountPerFeature": 255, "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, + "L1Regularization": 0.0000000002, "L2Regularization": 1, "LabelColumnName": "Label", "FeatureColumnName": "Feature" @@ -21,7 +21,7 @@ "NumberOfTrees": 4, "MaximumBinCountPerFeature": 255, "FeatureFraction": 1, - "LearningRate": 0.099999999999999978, + "LearningRate": 0.10, "LabelColumnName": "Label", "FeatureColumnName": "Feature" } diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt index fa86919915..d3be549eed 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndBinaryClassifiers.approved.txt @@ -13,7 +13,7 @@ "NumberOfTrees": 4, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, + "LearningRate": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -38,7 +38,7 @@ "SubsampleFraction": 1, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, + "L1Regularization": 0.0000000002, "L2Regularization": 1, "LabelColumnName": "Label", "FeatureColumnName": "Features" @@ -57,7 +57,7 @@ "estimatorType": "SdcaLogisticRegressionBinary", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt index 00a3d413c7..0e14cdb320 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndMultiClassifiers.approved.txt @@ -13,7 +13,7 @@ "NumberOfTrees": 4, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, + "LearningRate": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -38,7 +38,7 @@ "SubsampleFraction": 1, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, + "L1Regularization": 0.0000000002, "L2Regularization": 1, "LabelColumnName": "Label", "FeatureColumnName": "Features" @@ -66,7 +66,7 @@ "estimatorType": "SdcaMaximumEntropyMulti", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -75,7 +75,7 @@ "estimatorType": "SdcaLogisticRegressionOva", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt index 00a3d413c7..0e14cdb320 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromIEstimatorAndRegressors.approved.txt @@ -13,7 +13,7 @@ "NumberOfTrees": 4, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, + "LearningRate": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -38,7 +38,7 @@ "SubsampleFraction": 1, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, + "L1Regularization": 0.0000000002, "L2Regularization": 1, "LabelColumnName": "Label", "FeatureColumnName": "Features" @@ -66,7 +66,7 @@ "estimatorType": "SdcaMaximumEntropyMulti", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -75,7 +75,7 @@ "estimatorType": "SdcaLogisticRegressionOva", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt index d6f4d7d027..9661d5897c 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt @@ -19,7 +19,7 @@ "NumberOfTrees": 4, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, + "LearningRate": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -44,7 +44,7 @@ "SubsampleFraction": 1, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, + "L1Regularization": 0.0000000002, "L2Regularization": 1, "LabelColumnName": "Label", "FeatureColumnName": "Features" @@ -72,7 +72,7 @@ "estimatorType": "SdcaMaximumEntropyMulti", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -81,7 +81,7 @@ "estimatorType": "SdcaLogisticRegressionOva", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt index 21058f9cbd..05949bc69e 100644 --- a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt +++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableExtensionTest.CreateMultiModelPipelineFromSweepableEstimatorPipelineAndMultiClassifiers.approved.txt @@ -17,7 +17,7 @@ "NumberOfTrees": 4, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "LearningRate": 0.10000000000000001, + "LearningRate": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -42,7 +42,7 @@ "SubsampleFraction": 1, "MaximumBinCountPerFeature": 256, "FeatureFraction": 1, - "L1Regularization": 2.0000000000000001E-10, + "L1Regularization": 0.0000000002, "L2Regularization": 1, "LabelColumnName": "Label", "FeatureColumnName": "Features" @@ -70,7 +70,7 @@ "estimatorType": "SdcaMaximumEntropyMulti", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } @@ -79,7 +79,7 @@ "estimatorType": "SdcaLogisticRegressionOva", "parameter": { "L1Regularization": 1, - "L2Regularization": 0.100000001, + "L2Regularization": 0.1, "LabelColumnName": "Label", "FeatureColumnName": "Features" } diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs index 07e67ca3e6..1f78fd5126 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableEstimatorPipelineTest.cs @@ -15,6 +15,7 @@ using Xunit.Abstractions; using Microsoft.ML.AutoML.CodeGen; using System.Text.Json; +using System.Text.Json.Serialization; namespace Microsoft.ML.AutoML.Test { @@ -28,6 +29,10 @@ public SweepableEstimatorPipelineTest(ITestOutputHelper output) this._jsonSerializerOptions = new JsonSerializerOptions() { WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(), new DoubleToDecimalConverter(), new FloatToDecimalConverter(), + }, }; if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) diff --git a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs index e948d08696..9b93869672 100644 --- a/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs +++ b/test/Microsoft.ML.AutoML.Tests/SweepableExtensionTest.cs @@ -30,7 +30,7 @@ public SweepableExtensionTest(ITestOutputHelper output) WriteIndented = true, Converters = { - new JsonStringEnumConverter(), + new JsonStringEnumConverter(), new DoubleToDecimalConverter(), new FloatToDecimalConverter(), }, }; diff --git a/test/Microsoft.ML.AutoML.Tests/Utils/DoubleToDecimalConverter.cs b/test/Microsoft.ML.AutoML.Tests/Utils/DoubleToDecimalConverter.cs new file mode 100644 index 0000000000..02ef8cb868 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/Utils/DoubleToDecimalConverter.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Text.Json.Serialization; +using System.Text.Json; + +namespace Microsoft.ML.AutoML.Test +{ + internal class DoubleToDecimalConverter : JsonConverter + { + public override double Read(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options) + { + return Convert.ToDouble(reader.GetDecimal()); + } + + public override void Write(Utf8JsonWriter writer, double value, JsonSerializerOptions options) + { + writer.WriteNumberValue(Convert.ToDecimal(value)); + } + } +} diff --git a/test/Microsoft.ML.AutoML.Tests/Utils/FloatToDecimalConverter.cs b/test/Microsoft.ML.AutoML.Tests/Utils/FloatToDecimalConverter.cs new file mode 100644 index 0000000000..fc521f3200 --- /dev/null +++ b/test/Microsoft.ML.AutoML.Tests/Utils/FloatToDecimalConverter.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Text.Json.Serialization; +using System.Text.Json; + +namespace Microsoft.ML.AutoML.Test +{ + internal class FloatToDecimalConverter : JsonConverter + { + public override float Read(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options) + { + return Convert.ToSingle(reader.GetDecimal()); + } + + public override void Write(Utf8JsonWriter writer, float value, JsonSerializerOptions options) + { + writer.WriteNumberValue(Convert.ToDecimal(value)); + } + } +} From f3bc5ae9da19ab9ecf63e29ebce42ee555188410 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 28 Mar 2022 11:41:04 -0700 Subject: [PATCH 16/16] remove empty approved.txt --- ...ipelineTest.SingleModelPipeline_search_space_test.approved.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/SweepableEstimatorPipelineTest.SingleModelPipeline_search_space_test.approved.txt deleted file mode 100644 index e69de29bb2..0000000000