implement auto featurizer

src/Microsoft.ML.AutoML/API/AutoCatalog.cs

@@ -4,8 +4,11 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
 using Microsoft.ML.AutoML.CodeGen;
 using Microsoft.ML.Data;
+using Microsoft.ML.Runtime;
 using Microsoft.ML.SearchSpace;
 using Microsoft.ML.Trainers.FastTree;
 
@@ -538,55 +541,164 @@ public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNam
         /// <param name="inputColumnName">input column name.</param>
         internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inputColumnName)
         {
-            throw new NotImplementedException();
+            var option = new FeaturizeTextOption
+            {
+                InputColumnName = inputColumnName,
+                OutputColumnName = outputColumnName,
+            };
+
+            return new[] { SweepableEstimatorFactory.CreateFeaturizeText(option) };
         }
 
         /// <summary>
         /// Create a list of <see cref="SweepableEstimator"/> for featurizing numeric columns.
         /// </summary>
-        /// <param name="outputColumnName">output column name.</param>
-        /// <param name="inputColumnName">input column name.</param>
-        internal SweepableEstimator[] NumericFeaturizer(string outputColumnName, string inputColumnName)
+        /// <param name="outputColumnNames">output column names.</param>
+        /// <param name="inputColumnNames">input column names.</param>
+        internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, string[] inputColumnNames)
         {
-            throw new NotImplementedException();
+            Contracts.CheckValue(inputColumnNames, nameof(inputColumnNames));
+            Contracts.CheckValue(outputColumnNames, nameof(outputColumnNames));
+            Contracts.Check(outputColumnNames.Count() == inputColumnNames.Count() && outputColumnNames.Count() > 0, "outputColumnNames and inputColumnNames must have the same length and greater than 0");
+            var replaceMissingValueOption = new ReplaceMissingValueOption
+            {
+                InputColumnNames = inputColumnNames,
+                OutputColumnNames = outputColumnNames,
+            };
+
+            return new[] { SweepableEstimatorFactory.CreateReplaceMissingValues(replaceMissingValueOption) };
         }
 
         /// <summary>
         /// Create a list of <see cref="SweepableEstimator"/> for featurizing catalog columns.
         /// </summary>
-        /// <param name="outputColumnName">output column name.</param>
-        /// <param name="inputColumnName">input column name.</param>
-        internal SweepableEstimator[] CatalogFeaturizer(string outputColumnName, string inputColumnName)
+        /// <param name="outputColumnNames">output column names.</param>
+        /// <param name="inputColumnNames">input column names.</param>
+        internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, string[] inputColumnNames)
         {
-            throw new NotImplementedException();
+            Contracts.Check(outputColumnNames.Count() == inputColumnNames.Count() && outputColumnNames.Count() > 0, "outputColumnNames and inputColumnNames must have the same length and greater than 0");
+
+            var option = new OneHotOption
+            {
+                InputColumnNames = inputColumnNames,
+                OutputColumnNames = outputColumnNames,
+            };
+
+            return new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) };
         }
 
         /// <summary>
         /// Create a single featurize pipeline according to <paramref name="data"/>. This function will collect all columns in <paramref name="data"/> and not in <paramref name="excludeColumns"/>,
-        /// featurizing them using <see cref="CatalogFeaturizer(string, string)"/>, <see cref="NumericFeaturizer(string, string)"/> or <see cref="TextFeaturizer(string, string)"/>. And combine
+        /// featurizing them using <see cref="CatalogFeaturizer(string[], string[])"/>, <see cref="NumericFeaturizer(string[], string[])"/> or <see cref="TextFeaturizer(string, string)"/>. And combine
         /// them into a single feature column as output.
         /// </summary>
         /// <param name="data">input data.</param>
         /// <param name="catalogColumns">columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not.</param>
+        /// <param name="numericColumns">columns that should be treated as numeric. If not specified, it will automatically infer if a column is catalog or not.</param>
+        /// <param name="textColumns">columns that should be treated as text. If not specified, it will automatically infer if a column is catalog or not.</param>
         /// <param name="outputColumnName">output feature column.</param>
         /// <param name="excludeColumns">columns that won't be included when featurizing, like label</param>
-        internal MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] excludeColumns = null)
+        public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] excludeColumns = null)
         {
-            throw new NotImplementedException();
+            Contracts.CheckValue(data, nameof(data));
+
+            // validate if there's overlapping among catalogColumns, numericColumns, textColumns and excludeColumns
+            var overallColumns = new string[][] { catalogColumns, numericColumns, textColumns, excludeColumns }
+                                    .Where(c => c != null)
+                                    .SelectMany(c => c);
+
+            if (overallColumns != null)
+            {
+                Contracts.Assert(overallColumns.Count() == overallColumns.Distinct().Count(), "detect overlapping among catalogColumns, numericColumns, textColumns and excludedColumns");
+            }
+
+            var columnInfo = new ColumnInformation();
+
+            if (excludeColumns != null)
+            {
+                foreach (var ignoreColumn in excludeColumns)
+                {
+                    columnInfo.IgnoredColumnNames.Add(ignoreColumn);
+                }
+            }
+
+            if (catalogColumns != null)
+            {
+                foreach (var catalogColumn in catalogColumns)
+                {
+                    columnInfo.CategoricalColumnNames.Add(catalogColumn);
+                }
+            }
+
+            if (numericColumns != null)
+            {
+                foreach (var column in numericColumns)
+                {
+                    columnInfo.NumericColumnNames.Add(column);
+                }
+            }
+
+            if (textColumns != null)
+            {
+                foreach (var column in textColumns)
+                {
+                    columnInfo.TextColumnNames.Add(column);
+                }
+            }
+
+            return this.Featurizer(data, columnInfo, outputColumnName);
         }
 
         /// <summary>
-        /// Create a single featurize pipeline according to <paramref name="columnInformation"/>. This function will collect all columns in <paramref name="columnInformation"/> and not in <paramref name="excludeColumns"/>,
-        /// featurizing them using <see cref="CatalogFeaturizer(string, string)"/>, <see cref="NumericFeaturizer(string, string)"/> or <see cref="TextFeaturizer(string, string)"/>. And combine
+        /// Create a single featurize pipeline according to <paramref name="columnInformation"/>. This function will collect all columns in <paramref name="columnInformation"/>,
+        /// featurizing them using <see cref="CatalogFeaturizer(string[], string[])"/>, <see cref="NumericFeaturizer(string[], string[])"/> or <see cref="TextFeaturizer(string, string)"/>. And combine
         /// them into a single feature column as output.
         /// </summary>
+        /// <param name="data">input data.</param>
         /// <param name="columnInformation">column information.</param>
         /// <param name="outputColumnName">output feature column.</param>
-        /// <param name="excludeColumns">columns that won't be included when featurizing, like label</param>
-        /// <returns></returns>
-        internal MultiModelPipeline Featurizer(ColumnInformation columnInformation, string outputColumnName = "Features", string[] excludeColumns = null)
+        /// <returns>A <see cref="MultiModelPipeline"/> for featurization.</returns>
+        public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features")
         {
-            throw new NotImplementedException();
+            Contracts.CheckValue(data, nameof(data));
+            Contracts.CheckValue(columnInformation, nameof(columnInformation));
+
+            var columnPurposes = PurposeInference.InferPurposes(this._context, data, columnInformation);
+            var textFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.TextFeature);
+            var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature);
+            var catalogFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.CategoricalFeature);
+            var textFeatureColumnNames = textFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+            var numericFeatureColumnNames = numericFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+            var catalogFeatureColumnNames = catalogFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+
+            var pipeline = new MultiModelPipeline();
+            if (numericFeatureColumnNames.Length > 0)
+            {
+                pipeline = pipeline.Append(this.NumericFeaturizer(numericFeatureColumnNames, numericFeatureColumnNames));
+            }
+
+            if (catalogFeatureColumnNames.Length > 0)
+            {
+                pipeline = pipeline.Append(this.CatalogFeaturizer(catalogFeatureColumnNames, catalogFeatureColumnNames));
+            }
+
+            foreach (var textColumn in textFeatureColumnNames)
+            {
+                pipeline = pipeline.Append(this.TextFeaturizer(textColumn, textColumn));
+            }
+
+            var option = new ConcatOption
+            {
+                InputColumnNames = textFeatureColumnNames.Concat(numericFeatureColumnNames).Concat(catalogFeatureColumnNames).ToArray(),
+                OutputColumnName = outputColumnName,
+            };
+
+            if (option.InputColumnNames.Length > 0)
+            {
+                pipeline = pipeline.Append(SweepableEstimatorFactory.CreateConcatenate(option));
+            }
+
+            return pipeline;
         }
     }
 }

src/Microsoft.ML.Data/Transforms/Hashing.cs

@@ -182,7 +182,7 @@ internal HashingTransformer(IHostEnvironment env, params HashingEstimator.Column
             foreach (var column in _columns)
             {
                 if (column.MaximumNumberOfInverts != 0)
-                    throw Host.ExceptParam(nameof(columns), $"Found column with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use { nameof(HashingEstimator)} instead");
+                    throw Host.ExceptParam(nameof(columns), $"Found column with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use {nameof(HashingEstimator)} instead");
 
                 if (column.Combine && column.UseOrderedHashing)
                     throw Host.ExceptParam(nameof(HashingEstimator.ColumnOptions.Combine), "When the 'Combine' option is specified, ordered hashing is not supported.");

src/Microsoft.ML.Transforms/Text/NgramHashingTransformer.cs

@@ -184,7 +184,7 @@ internal NgramHashingTransformer(IHostEnvironment env, params NgramHashingEstima
             foreach (var column in _columns)
             {
                 if (column.MaximumNumberOfInverts != 0)
-                    throw Host.ExceptParam(nameof(columns), $"Found colunm with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use { nameof(NgramHashingEstimator)} instead");
+                    throw Host.ExceptParam(nameof(columns), $"Found colunm with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use {nameof(NgramHashingEstimator)} instead");
             }
         }
 

test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt

@@ -0,0 +1,34 @@
+{
+  "schema": "e0 * e1",
+  "estimators": {
+    "e0": {
+      "estimatorType": "ReplaceMissingValues",
+      "parameter": {
+        "OutputColumnNames": [
+          "col1",
+          "col2",
+          "col3",
+          "col4"
+        ],
+        "InputColumnNames": [
+          "col1",
+          "col2",
+          "col3",
+          "col4"
+        ]
+      }
+    },
+    "e1": {
+      "estimatorType": "Concatenate",
+      "parameter": {
+        "InputColumnNames": [
+          "col1",
+          "col2",
+          "col3",
+          "col4"
+        ],
+        "OutputColumnName": "Features"
+      }
+    }
+  }
+}

test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt

@@ -0,0 +1,83 @@
+{
+  "schema": "e0 * (e1 \u002B e2) * e3",
+  "estimators": {
+    "e0": {
+      "estimatorType": "ReplaceMissingValues",
+      "parameter": {
+        "OutputColumnNames": [
+          "Features"
+        ],
+        "InputColumnNames": [
+          "Features"
+        ]
+      }
+    },
+    "e1": {
+      "estimatorType": "OneHotEncoding",
+      "parameter": {
+        "OutputColumnNames": [
+          "Workclass",
+          "education",
+          "marital-status",
+          "occupation",
+          "relationship",
+          "ethnicity",
+          "sex",
+          "native-country-region"
+        ],
+        "InputColumnNames": [
+          "Workclass",
+          "education",
+          "marital-status",
+          "occupation",
+          "relationship",
+          "ethnicity",
+          "sex",
+          "native-country-region"
+        ]
+      }
+    },
+    "e2": {
+      "estimatorType": "OneHotHashEncoding",
+      "parameter": {
+        "OutputColumnNames": [
+          "Workclass",
+          "education",
+          "marital-status",
+          "occupation",
+          "relationship",
+          "ethnicity",
+          "sex",
+          "native-country-region"
+        ],
+        "InputColumnNames": [
+          "Workclass",
+          "education",
+          "marital-status",
+          "occupation",
+          "relationship",
+          "ethnicity",
+          "sex",
+          "native-country-region"
+        ]
+      }
+    },
+    "e3": {
+      "estimatorType": "Concatenate",
+      "parameter": {
+        "InputColumnNames": [
+          "Features",
+          "Workclass",
+          "education",
+          "marital-status",
+          "occupation",
+          "relationship",
+          "ethnicity",
+          "sex",
+          "native-country-region"
+        ],
+        "OutputColumnName": "OutputFeature"
+      }
+    }
+  }
+}

test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs

@@ -0,0 +1,65 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Text.Json;
+using Microsoft.ML.TestFramework;
+using Xunit;
+using Xunit.Abstractions;
+using ApprovalTests;
+using ApprovalTests.Namers;
+using ApprovalTests.Reporters;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.ML.AutoML.Test
+{
+    public class AutoFeaturizerTests : BaseTestClass
+    {
+        private readonly JsonSerializerOptions _jsonSerializerOptions;
+
+        public AutoFeaturizerTests(ITestOutputHelper output)
+            : base(output)
+        {
+            _jsonSerializerOptions = new JsonSerializerOptions()
+            {
+                WriteIndented = true,
+                Converters =
+                {
+                    new JsonStringEnumConverter(), new DoubleToDecimalConverter(), new FloatToDecimalConverter(),
+                },
+            };
+
+            if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null)
+            {
+                Approvals.UseAssemblyLocationForApprovedFiles();
+            }
+        }
+
+        [Fact]
+        [UseReporter(typeof(DiffReporter))]
+        [UseApprovalSubdirectory("ApprovalTests")]
+        public void AutoFeaturizer_uci_adult_test()
+        {
+            var context = new MLContext(1);
+            var dataset = DatasetUtil.GetUciAdultDataView();
+            var pipeline = context.Auto().Featurizer(dataset, outputColumnName: "OutputFeature", excludeColumns: new[] { "Label" });
+
+            Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions));
+        }
+
+        [Fact]
+        [UseReporter(typeof(DiffReporter))]
+        [UseApprovalSubdirectory("ApprovalTests")]
+        public void AutoFeaturizer_iris_test()
+        {
+            var context = new MLContext(1);
+            var dataset = DatasetUtil.GetIrisDataView();
+            var pipeline = context.Auto().Featurizer(dataset, excludeColumns: new[] { "Label" });
+
+            Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions));
+        }
+    }
+}