diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs
index 6937e4e59e..1f78d5e0c3 100644
--- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs
+++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs
@@ -4,8 +4,11 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
using Microsoft.ML.AutoML.CodeGen;
using Microsoft.ML.Data;
+using Microsoft.ML.Runtime;
using Microsoft.ML.SearchSpace;
using Microsoft.ML.Trainers.FastTree;
@@ -538,55 +541,164 @@ public SweepableEstimator[] Regression(string labelColumnName = DefaultColumnNam
/// input column name.
internal SweepableEstimator[] TextFeaturizer(string outputColumnName, string inputColumnName)
{
- throw new NotImplementedException();
+ var option = new FeaturizeTextOption
+ {
+ InputColumnName = inputColumnName,
+ OutputColumnName = outputColumnName,
+ };
+
+ return new[] { SweepableEstimatorFactory.CreateFeaturizeText(option) };
}
///
/// Create a list of for featurizing numeric columns.
///
- /// output column name.
- /// input column name.
- internal SweepableEstimator[] NumericFeaturizer(string outputColumnName, string inputColumnName)
+ /// output column names.
+ /// input column names.
+ internal SweepableEstimator[] NumericFeaturizer(string[] outputColumnNames, string[] inputColumnNames)
{
- throw new NotImplementedException();
+ Contracts.CheckValue(inputColumnNames, nameof(inputColumnNames));
+ Contracts.CheckValue(outputColumnNames, nameof(outputColumnNames));
+ Contracts.Check(outputColumnNames.Count() == inputColumnNames.Count() && outputColumnNames.Count() > 0, "outputColumnNames and inputColumnNames must have the same length and greater than 0");
+ var replaceMissingValueOption = new ReplaceMissingValueOption
+ {
+ InputColumnNames = inputColumnNames,
+ OutputColumnNames = outputColumnNames,
+ };
+
+ return new[] { SweepableEstimatorFactory.CreateReplaceMissingValues(replaceMissingValueOption) };
}
///
/// Create a list of for featurizing catalog columns.
///
- /// output column name.
- /// input column name.
- internal SweepableEstimator[] CatalogFeaturizer(string outputColumnName, string inputColumnName)
+ /// output column names.
+ /// input column names.
+ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, string[] inputColumnNames)
{
- throw new NotImplementedException();
+ Contracts.Check(outputColumnNames.Count() == inputColumnNames.Count() && outputColumnNames.Count() > 0, "outputColumnNames and inputColumnNames must have the same length and greater than 0");
+
+ var option = new OneHotOption
+ {
+ InputColumnNames = inputColumnNames,
+ OutputColumnNames = outputColumnNames,
+ };
+
+ return new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) };
}
///
/// Create a single featurize pipeline according to . This function will collect all columns in and not in ,
- /// featurizing them using , or . And combine
+ /// featurizing them using , or . And combine
/// them into a single feature column as output.
///
/// input data.
/// columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not.
+ /// columns that should be treated as numeric. If not specified, it will automatically infer if a column is catalog or not.
+ /// columns that should be treated as text. If not specified, it will automatically infer if a column is catalog or not.
/// output feature column.
/// columns that won't be included when featurizing, like label
- internal MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] excludeColumns = null)
+ public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] excludeColumns = null)
{
- throw new NotImplementedException();
+ Contracts.CheckValue(data, nameof(data));
+
+ // validate if there's overlapping among catalogColumns, numericColumns, textColumns and excludeColumns
+ var overallColumns = new string[][] { catalogColumns, numericColumns, textColumns, excludeColumns }
+ .Where(c => c != null)
+ .SelectMany(c => c);
+
+ if (overallColumns != null)
+ {
+ Contracts.Assert(overallColumns.Count() == overallColumns.Distinct().Count(), "detect overlapping among catalogColumns, numericColumns, textColumns and excludedColumns");
+ }
+
+ var columnInfo = new ColumnInformation();
+
+ if (excludeColumns != null)
+ {
+ foreach (var ignoreColumn in excludeColumns)
+ {
+ columnInfo.IgnoredColumnNames.Add(ignoreColumn);
+ }
+ }
+
+ if (catalogColumns != null)
+ {
+ foreach (var catalogColumn in catalogColumns)
+ {
+ columnInfo.CategoricalColumnNames.Add(catalogColumn);
+ }
+ }
+
+ if (numericColumns != null)
+ {
+ foreach (var column in numericColumns)
+ {
+ columnInfo.NumericColumnNames.Add(column);
+ }
+ }
+
+ if (textColumns != null)
+ {
+ foreach (var column in textColumns)
+ {
+ columnInfo.TextColumnNames.Add(column);
+ }
+ }
+
+ return this.Featurizer(data, columnInfo, outputColumnName);
}
///
- /// Create a single featurize pipeline according to . This function will collect all columns in and not in ,
- /// featurizing them using , or . And combine
+ /// Create a single featurize pipeline according to . This function will collect all columns in ,
+ /// featurizing them using , or . And combine
/// them into a single feature column as output.
///
+ /// input data.
/// column information.
/// output feature column.
- /// columns that won't be included when featurizing, like label
- ///
- internal MultiModelPipeline Featurizer(ColumnInformation columnInformation, string outputColumnName = "Features", string[] excludeColumns = null)
+ /// A for featurization.
+ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInformation, string outputColumnName = "Features")
{
- throw new NotImplementedException();
+ Contracts.CheckValue(data, nameof(data));
+ Contracts.CheckValue(columnInformation, nameof(columnInformation));
+
+ var columnPurposes = PurposeInference.InferPurposes(this._context, data, columnInformation);
+ var textFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.TextFeature);
+ var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature);
+ var catalogFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.CategoricalFeature);
+ var textFeatureColumnNames = textFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+ var numericFeatureColumnNames = numericFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+ var catalogFeatureColumnNames = catalogFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+
+ var pipeline = new MultiModelPipeline();
+ if (numericFeatureColumnNames.Length > 0)
+ {
+ pipeline = pipeline.Append(this.NumericFeaturizer(numericFeatureColumnNames, numericFeatureColumnNames));
+ }
+
+ if (catalogFeatureColumnNames.Length > 0)
+ {
+ pipeline = pipeline.Append(this.CatalogFeaturizer(catalogFeatureColumnNames, catalogFeatureColumnNames));
+ }
+
+ foreach (var textColumn in textFeatureColumnNames)
+ {
+ pipeline = pipeline.Append(this.TextFeaturizer(textColumn, textColumn));
+ }
+
+ var option = new ConcatOption
+ {
+ InputColumnNames = textFeatureColumnNames.Concat(numericFeatureColumnNames).Concat(catalogFeatureColumnNames).ToArray(),
+ OutputColumnName = outputColumnName,
+ };
+
+ if (option.InputColumnNames.Length > 0)
+ {
+ pipeline = pipeline.Append(SweepableEstimatorFactory.CreateConcatenate(option));
+ }
+
+ return pipeline;
}
}
}
diff --git a/src/Microsoft.ML.Data/Transforms/Hashing.cs b/src/Microsoft.ML.Data/Transforms/Hashing.cs
index 8e7dd10221..d3131ea2c1 100644
--- a/src/Microsoft.ML.Data/Transforms/Hashing.cs
+++ b/src/Microsoft.ML.Data/Transforms/Hashing.cs
@@ -182,7 +182,7 @@ internal HashingTransformer(IHostEnvironment env, params HashingEstimator.Column
foreach (var column in _columns)
{
if (column.MaximumNumberOfInverts != 0)
- throw Host.ExceptParam(nameof(columns), $"Found column with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use { nameof(HashingEstimator)} instead");
+ throw Host.ExceptParam(nameof(columns), $"Found column with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use {nameof(HashingEstimator)} instead");
if (column.Combine && column.UseOrderedHashing)
throw Host.ExceptParam(nameof(HashingEstimator.ColumnOptions.Combine), "When the 'Combine' option is specified, ordered hashing is not supported.");
diff --git a/src/Microsoft.ML.Transforms/Text/NgramHashingTransformer.cs b/src/Microsoft.ML.Transforms/Text/NgramHashingTransformer.cs
index 8b75eb0d3c..484f6060c2 100644
--- a/src/Microsoft.ML.Transforms/Text/NgramHashingTransformer.cs
+++ b/src/Microsoft.ML.Transforms/Text/NgramHashingTransformer.cs
@@ -184,7 +184,7 @@ internal NgramHashingTransformer(IHostEnvironment env, params NgramHashingEstima
foreach (var column in _columns)
{
if (column.MaximumNumberOfInverts != 0)
- throw Host.ExceptParam(nameof(columns), $"Found colunm with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use { nameof(NgramHashingEstimator)} instead");
+ throw Host.ExceptParam(nameof(columns), $"Found colunm with {nameof(column.MaximumNumberOfInverts)} set to non zero value, please use {nameof(NgramHashingEstimator)} instead");
}
}
diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt
new file mode 100644
index 0000000000..e6ec8ed089
--- /dev/null
+++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_iris_test.approved.txt
@@ -0,0 +1,34 @@
+{
+ "schema": "e0 * e1",
+ "estimators": {
+ "e0": {
+ "estimatorType": "ReplaceMissingValues",
+ "parameter": {
+ "OutputColumnNames": [
+ "col1",
+ "col2",
+ "col3",
+ "col4"
+ ],
+ "InputColumnNames": [
+ "col1",
+ "col2",
+ "col3",
+ "col4"
+ ]
+ }
+ },
+ "e1": {
+ "estimatorType": "Concatenate",
+ "parameter": {
+ "InputColumnNames": [
+ "col1",
+ "col2",
+ "col3",
+ "col4"
+ ],
+ "OutputColumnName": "Features"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt
new file mode 100644
index 0000000000..63a64233a4
--- /dev/null
+++ b/test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_uci_adult_test.approved.txt
@@ -0,0 +1,83 @@
+{
+ "schema": "e0 * (e1 \u002B e2) * e3",
+ "estimators": {
+ "e0": {
+ "estimatorType": "ReplaceMissingValues",
+ "parameter": {
+ "OutputColumnNames": [
+ "Features"
+ ],
+ "InputColumnNames": [
+ "Features"
+ ]
+ }
+ },
+ "e1": {
+ "estimatorType": "OneHotEncoding",
+ "parameter": {
+ "OutputColumnNames": [
+ "Workclass",
+ "education",
+ "marital-status",
+ "occupation",
+ "relationship",
+ "ethnicity",
+ "sex",
+ "native-country-region"
+ ],
+ "InputColumnNames": [
+ "Workclass",
+ "education",
+ "marital-status",
+ "occupation",
+ "relationship",
+ "ethnicity",
+ "sex",
+ "native-country-region"
+ ]
+ }
+ },
+ "e2": {
+ "estimatorType": "OneHotHashEncoding",
+ "parameter": {
+ "OutputColumnNames": [
+ "Workclass",
+ "education",
+ "marital-status",
+ "occupation",
+ "relationship",
+ "ethnicity",
+ "sex",
+ "native-country-region"
+ ],
+ "InputColumnNames": [
+ "Workclass",
+ "education",
+ "marital-status",
+ "occupation",
+ "relationship",
+ "ethnicity",
+ "sex",
+ "native-country-region"
+ ]
+ }
+ },
+ "e3": {
+ "estimatorType": "Concatenate",
+ "parameter": {
+ "InputColumnNames": [
+ "Features",
+ "Workclass",
+ "education",
+ "marital-status",
+ "occupation",
+ "relationship",
+ "ethnicity",
+ "sex",
+ "native-country-region"
+ ],
+ "OutputColumnName": "OutputFeature"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs
new file mode 100644
index 0000000000..aa66207921
--- /dev/null
+++ b/test/Microsoft.ML.AutoML.Tests/AutoFeaturizerTests.cs
@@ -0,0 +1,65 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Text.Json;
+using Microsoft.ML.TestFramework;
+using Xunit;
+using Xunit.Abstractions;
+using ApprovalTests;
+using ApprovalTests.Namers;
+using ApprovalTests.Reporters;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.ML.AutoML.Test
+{
+ public class AutoFeaturizerTests : BaseTestClass
+ {
+ private readonly JsonSerializerOptions _jsonSerializerOptions;
+
+ public AutoFeaturizerTests(ITestOutputHelper output)
+ : base(output)
+ {
+ _jsonSerializerOptions = new JsonSerializerOptions()
+ {
+ WriteIndented = true,
+ Converters =
+ {
+ new JsonStringEnumConverter(), new DoubleToDecimalConverter(), new FloatToDecimalConverter(),
+ },
+ };
+
+ if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null)
+ {
+ Approvals.UseAssemblyLocationForApprovedFiles();
+ }
+ }
+
+ [Fact]
+ [UseReporter(typeof(DiffReporter))]
+ [UseApprovalSubdirectory("ApprovalTests")]
+ public void AutoFeaturizer_uci_adult_test()
+ {
+ var context = new MLContext(1);
+ var dataset = DatasetUtil.GetUciAdultDataView();
+ var pipeline = context.Auto().Featurizer(dataset, outputColumnName: "OutputFeature", excludeColumns: new[] { "Label" });
+
+ Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions));
+ }
+
+ [Fact]
+ [UseReporter(typeof(DiffReporter))]
+ [UseApprovalSubdirectory("ApprovalTests")]
+ public void AutoFeaturizer_iris_test()
+ {
+ var context = new MLContext(1);
+ var dataset = DatasetUtil.GetIrisDataView();
+ var pipeline = context.Auto().Featurizer(dataset, excludeColumns: new[] { "Label" });
+
+ Approvals.Verify(JsonSerializer.Serialize(pipeline, _jsonSerializerOptions));
+ }
+ }
+}
diff --git a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs
index a465e55fcd..17cdfb2791 100644
--- a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs
+++ b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs
@@ -24,6 +24,8 @@ internal static class DatasetUtil
private static IDataView _uciAdultDataView;
+ private static IDataView _irisDataView;
+
public static string GetUciAdultDataset() => GetDataPath("adult.tiny.with-schema.txt");
public static string GetMlNetGeneratedRegressionDataset() => GetDataPath("generated_regression_dataset.csv");
@@ -50,6 +52,19 @@ public static IDataView GetUciAdultDataView()
return _uciAdultDataView;
}
+ public static IDataView GetIrisDataView()
+ {
+ if (_irisDataView == null)
+ {
+ var context = new MLContext(1);
+ var dataFile = GetIrisDataset();
+ var columnInferenceResult = context.Auto().InferColumns(dataFile, 0, groupColumns: false);
+ var textLoader = context.Data.CreateTextLoader(columnInferenceResult.TextLoaderOptions);
+ _irisDataView = textLoader.Load(dataFile);
+ }
+ return _irisDataView;
+ }
+
public static string GetFlowersDataset()
{
const string datasetName = @"flowers";
diff --git a/test/Microsoft.ML.AutoML.Tests/GridSearchTunerTests.cs b/test/Microsoft.ML.AutoML.Tests/GridSearchTunerTests.cs
index b59aada665..fab4f0de67 100644
--- a/test/Microsoft.ML.AutoML.Tests/GridSearchTunerTests.cs
+++ b/test/Microsoft.ML.AutoML.Tests/GridSearchTunerTests.cs
@@ -12,7 +12,7 @@
using Xunit;
using Xunit.Abstractions;
-namespace Microsoft.ML.AutoML.Tests
+namespace Microsoft.ML.AutoML.Test
{
public class GridSearchTunerTests : BaseTestClass
{