From 41e7d362921d1ef0fc1088bea08a9ea1b7a48b31 Mon Sep 17 00:00:00 2001 From: Daniel Holstein Date: Wed, 20 Nov 2019 15:05:50 -0800 Subject: [PATCH 1/2] Ignore hidden columns in AutoML schema checks of validation data --- .../Utils/UserInputValidationUtil.cs | 7 ++++- .../UserInputValidationTests.cs | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs b/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs index 594b3eaeec..186c5aeb49 100644 --- a/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs @@ -183,7 +183,7 @@ private static void ValidateValidationData(IDataView trainData, IDataView valida const string schemaMismatchError = "Training data and validation data schemas do not match."; - if (trainData.Schema.Count != validationData.Schema.Count) + if (trainData.Schema.Count(c => !c.IsHidden) != validationData.Schema.Count(c => !c.IsHidden)) { throw new ArgumentException($"{schemaMismatchError} Train data has '{trainData.Schema.Count}' columns," + $"and validation data has '{validationData.Schema.Count}' columns.", nameof(validationData)); @@ -191,6 +191,11 @@ private static void ValidateValidationData(IDataView trainData, IDataView valida foreach (var trainCol in trainData.Schema) { + if (trainCol.IsHidden) + { + continue; + } + var validCol = validationData.Schema.GetColumnOrNull(trainCol.Name); if (validCol == null) { diff --git a/test/Microsoft.ML.AutoML.Tests/UserInputValidationTests.cs b/test/Microsoft.ML.AutoML.Tests/UserInputValidationTests.cs index 2236cc4a21..79c9a93b4c 100644 --- a/test/Microsoft.ML.AutoML.Tests/UserInputValidationTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/UserInputValidationTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.IO; using Microsoft.ML.Data; using Xunit; @@ -307,6 +308,34 @@ public void ValidateEmptyValidationDataThrows() } } + + [Fact] + public void TestValidationDataSchemaChecksIgnoreHiddenColumns() + { + var mlContext = new MLContext(); + + // Build training data where label column is a float. + var trainDataBuilder = new ArrayDataViewBuilder(mlContext); + trainDataBuilder.AddColumn("Number", NumberDataViewType.Single, 0f); + trainDataBuilder.AddColumn(DefaultColumnNames.Label, NumberDataViewType.Single, 0f); + var trainingData = trainDataBuilder.GetDataView(); + + // In the training data, transform the label column from a float to a Boolean. This has the effect of + // creating a hidden column named 'Label' of type float and an additional column named 'Label' of type Boolean. + var convertLabelToBoolEstimator = mlContext.Transforms.Conversion.MapValue(DefaultColumnNames.Label, + new List>() { new KeyValuePair(1, true) }); + trainingData = convertLabelToBoolEstimator.Fit(trainingData).Transform(trainingData); + + // Build validaiton data where label column is a Boolean. + var validationDataBuilder = new ArrayDataViewBuilder(mlContext); + validationDataBuilder.AddColumn("Number", NumberDataViewType.Single, 0f); + validationDataBuilder.AddColumn(DefaultColumnNames.Label, BooleanDataViewType.Instance, false); + var validationData = validationDataBuilder.GetDataView(); + + UserInputValidationUtil.ValidateExperimentExecuteArgs(trainingData, new ColumnInformation(), validationData, TaskKind.BinaryClassification); + } + + private static void ValidateLabelTypeTestCore(TaskKind task, PrimitiveDataViewType labelType, bool labelTypeShouldBeValid) { var dataViewBuilder = new ArrayDataViewBuilder(new MLContext()); From e24d565c02b15fd9da7ab78c6009e1e98ef3d128 Mon Sep 17 00:00:00 2001 From: Daniel Holstein Date: Thu, 21 Nov 2019 11:11:44 -0800 Subject: [PATCH 2/2] comment rev --- src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs b/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs index 186c5aeb49..368eaabd59 100644 --- a/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs @@ -189,6 +189,9 @@ private static void ValidateValidationData(IDataView trainData, IDataView valida $"and validation data has '{validationData.Schema.Count}' columns.", nameof(validationData)); } + // Validate that every active column in the train data corresponds to an active column in the validation data. + // (Indirectly, since we asserted above that the train and validation data have the same number of active columns, this also + // esnures the reverse -- that every active column in the validation data corresponds to an active column in the train data.) foreach (var trainCol in trainData.Schema) { if (trainCol.IsHidden)