From e9baae9d1ad85e993d50a664b2d8d6431fac4e7a Mon Sep 17 00:00:00 2001 From: Piotr Telman Date: Mon, 25 May 2020 15:16:13 +0200 Subject: [PATCH 01/13] Fix issue when parsing float string fails on pl-PL culture set --- .../Sweepers/SweeperProbabilityUtils.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs b/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs index f848a4e27b..b17bd40e9d 100644 --- a/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs +++ b/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs @@ -98,13 +98,15 @@ public static float[] ParameterSetAsFloatArray(IValueGenerator[] sweepParams, Pa } else if (sweepParam is LongValueGenerator lvg) { + var longValue = GetIfIParameterValueOfT(pset) ?? long.Parse(pset.ValueText); // Normalizing all numeric parameters to [0,1] range. - result.Add(lvg.NormalizeValue(new LongParameterValue(pset.Name, long.Parse(pset.ValueText)))); + result.Add(lvg.NormalizeValue(new LongParameterValue(pset.Name, longValue))); } else if (sweepParam is FloatValueGenerator fvg) { + var floatValue = GetIfIParameterValueOfT(pset) ?? float.Parse(pset.ValueText); // Normalizing all numeric parameters to [0,1] range. - result.Add(fvg.NormalizeValue(new FloatParameterValue(pset.Name, float.Parse(pset.ValueText)))); + result.Add(fvg.NormalizeValue(new FloatParameterValue(pset.Name, floatValue))); } else { @@ -115,6 +117,10 @@ public static float[] ParameterSetAsFloatArray(IValueGenerator[] sweepParams, Pa return result.ToArray(); } + private static T? GetIfIParameterValueOfT(IParameterValue parameterValue) + where T : struct => + parameterValue is IParameterValue pvt ? pvt.Value : default(T?); + public static ParameterSet FloatArrayAsParameterSet(IValueGenerator[] sweepParams, float[] array, bool expandedCategoricals = true) { Runtime.Contracts.Assert(array.Length == sweepParams.Length); From e58dd1511668f0527cd5ea4fd53d1ad7665d716b Mon Sep 17 00:00:00 2001 From: Piotr Telman Date: Tue, 26 May 2020 07:15:01 +0200 Subject: [PATCH 02/13] Added InvariantCulture float parsing as per CodeReview request --- src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs b/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs index b17bd40e9d..510755587f 100644 --- a/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs +++ b/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using Microsoft.ML.Internal.CpuMath; namespace Microsoft.ML.AutoML @@ -104,7 +105,7 @@ public static float[] ParameterSetAsFloatArray(IValueGenerator[] sweepParams, Pa } else if (sweepParam is FloatValueGenerator fvg) { - var floatValue = GetIfIParameterValueOfT(pset) ?? float.Parse(pset.ValueText); + var floatValue = GetIfIParameterValueOfT(pset) ?? float.Parse(pset.ValueText, CultureInfo.InvariantCulture); // Normalizing all numeric parameters to [0,1] range. result.Add(fvg.NormalizeValue(new FloatParameterValue(pset.Name, floatValue))); } From a956afb110ef5fff1fcb7186c7eb169ef4e08c4b Mon Sep 17 00:00:00 2001 From: Piotr Telman Date: Wed, 28 Oct 2020 12:31:52 +0100 Subject: [PATCH 03/13] Update src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs Co-authored-by: Justin Ormont --- src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs b/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs index 510755587f..89c13c152f 100644 --- a/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs +++ b/src/Microsoft.ML.AutoML/Sweepers/SweeperProbabilityUtils.cs @@ -99,7 +99,7 @@ public static float[] ParameterSetAsFloatArray(IValueGenerator[] sweepParams, Pa } else if (sweepParam is LongValueGenerator lvg) { - var longValue = GetIfIParameterValueOfT(pset) ?? long.Parse(pset.ValueText); + var longValue = GetIfIParameterValueOfT(pset) ?? long.Parse(pset.ValueText, CultureInfo.InvariantCulture); // Normalizing all numeric parameters to [0,1] range. result.Add(lvg.NormalizeValue(new LongParameterValue(pset.Name, longValue))); } From 3724f551d54139de677039e29ccbc0d3ac6b3356 Mon Sep 17 00:00:00 2001 From: Piotr Telman Date: Wed, 28 Oct 2020 13:38:24 +0100 Subject: [PATCH 04/13] Update Parameters.cs --- src/Microsoft.ML.AutoML/Sweepers/Parameters.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Sweepers/Parameters.cs b/src/Microsoft.ML.AutoML/Sweepers/Parameters.cs index 2bf22db4c1..54765684d0 100644 --- a/src/Microsoft.ML.AutoML/Sweepers/Parameters.cs +++ b/src/Microsoft.ML.AutoML/Sweepers/Parameters.cs @@ -83,7 +83,7 @@ public LongParameterValue(string name, long value) { _name = name; _value = value; - _valueText = _value.ToString("D"); + _valueText = _value.ToString("D", CultureInfo.InvariantCulture); } public bool Equals(IParameterValue other) From 32f931eed09c56c1dd3d8028097d33cb9d781cd7 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Wed, 28 Oct 2020 15:35:22 -0700 Subject: [PATCH 05/13] Added PL test --- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 40ccfdc067..b77700fa01 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Globalization; using System.Linq; +using System.Threading; using Microsoft.ML.Data; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; @@ -102,9 +104,27 @@ private void Context_Log(object sender, LoggingEventArgs e) //throw new NotImplementedException(); } - [Fact] - public void AutoFitRegressionTest() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AutoFitRegressionTest(bool foreignCulture) { + var originalCulture = Thread.CurrentThread.CurrentCulture; + uint experimentTime = 0; + + if (foreignCulture) + { + // If users run AutoML with a different local, sometimes + // the sweeper encounters problems when parsing some strings. + // So testing in another culture is necessary. + // Furthermore, these issues might only occur after several + // iterations, so more experiment time is needed for this to + // occur. + Thread.CurrentThread.CurrentCulture = new CultureInfo("pl-PL"); + experimentTime = 30; + + } + var context = new MLContext(1); var dataPath = DatasetUtil.GetMlNetGeneratedRegressionDataset(); var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.MlNetGeneratedRegressionLabel); @@ -113,11 +133,14 @@ public void AutoFitRegressionTest() var validationData = context.Data.TakeRows(trainData, 20); trainData = context.Data.SkipRows(trainData, 20); var result = context.Auto() - .CreateRegressionExperiment(0) + .CreateRegressionExperiment(experimentTime) .Execute(trainData, validationData, new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); + + if(foreignCulture) + Thread.CurrentThread.CurrentCulture = originalCulture; } [LightGBMFact] @@ -351,4 +374,4 @@ private TextLoader.Options GetLoaderArgsRank(string labelColumnName, string grou }; } } -} +} \ No newline at end of file From ff655708bafbf8d0e98dd52bb0803be4fbee7b39 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Wed, 28 Oct 2020 17:08:09 -0700 Subject: [PATCH 06/13] Added multiple cultures --- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index b77700fa01..c4e169d387 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -105,14 +105,17 @@ private void Context_Log(object sender, LoggingEventArgs e) } [Theory] - [InlineData(false)] - [InlineData(true)] - public void AutoFitRegressionTest(bool foreignCulture) + [InlineData("en-US")] + [InlineData("ar-SA")] + [InlineData("pl-PL")] + public void AutoFitRegressionTest(string culture) { var originalCulture = Thread.CurrentThread.CurrentCulture; + Thread.CurrentThread.CurrentCulture = new CultureInfo(culture); + uint experimentTime = 0; - if (foreignCulture) + if (culture == "ar-SA") { // If users run AutoML with a different local, sometimes // the sweeper encounters problems when parsing some strings. @@ -120,11 +123,14 @@ public void AutoFitRegressionTest(bool foreignCulture) // Furthermore, these issues might only occur after several // iterations, so more experiment time is needed for this to // occur. - Thread.CurrentThread.CurrentCulture = new CultureInfo("pl-PL"); experimentTime = 30; } - + else if(culture == "pl-PL") + { + experimentTime = 100; + } + var context = new MLContext(1); var dataPath = DatasetUtil.GetMlNetGeneratedRegressionDataset(); var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.MlNetGeneratedRegressionLabel); @@ -139,8 +145,7 @@ public void AutoFitRegressionTest(bool foreignCulture) Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); - if(foreignCulture) - Thread.CurrentThread.CurrentCulture = originalCulture; + Thread.CurrentThread.CurrentCulture = originalCulture; } [LightGBMFact] From 4f6b119a04d280262c01e417337e9bf9060ee80a Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Wed, 28 Oct 2020 20:01:46 -0700 Subject: [PATCH 07/13] debugging CI failure --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index c4e169d387..f2c23f679f 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -143,6 +143,9 @@ public void AutoFitRegressionTest(string culture) .Execute(trainData, validationData, new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); + //MYTODO: Only adding this for debugging purposes on the CI: + System.Console.WriteLine($"culture:{culture} - Count: {result.RunDetails.Count()} - Null ValidationMetrics Count:{result.RunDetails.Where(rd => rd.ValidationMetrics == null).Count()}"); + Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); Thread.CurrentThread.CurrentCulture = originalCulture; From 95b728099415cacbe8cf3819ec51ce50cec94eb2 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 29 Oct 2020 12:00:38 -0700 Subject: [PATCH 08/13] Debug runSpecific --- .vsts-dotnet-ci.yml | 14 +++++++------- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 13 +++++++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 5c2dcf30cb..9046e50b1d 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,7 +27,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -36,7 +36,7 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: true + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -44,7 +44,7 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: true + runSpecific: true pool: name: Hosted macOS @@ -63,7 +63,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -72,7 +72,7 @@ jobs: parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - innerLoop: true + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -92,7 +92,7 @@ jobs: _config_short: RFX _includeBenchmarkData: false _targetFramework: win-x64 - innerLoop: true + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -102,7 +102,7 @@ jobs: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - innerLoop: true + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index f2c23f679f..9ebee2f7df 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Globalization; using System.Linq; using System.Threading; @@ -105,6 +106,7 @@ private void Context_Log(object sender, LoggingEventArgs e) } [Theory] + [Trait("Category", "RunSpecificTest")] [InlineData("en-US")] [InlineData("ar-SA")] [InlineData("pl-PL")] @@ -113,7 +115,7 @@ public void AutoFitRegressionTest(string culture) var originalCulture = Thread.CurrentThread.CurrentCulture; Thread.CurrentThread.CurrentCulture = new CultureInfo(culture); - uint experimentTime = 0; + uint experimentTime = 30; if (culture == "ar-SA") { @@ -132,6 +134,7 @@ public void AutoFitRegressionTest(string culture) } var context = new MLContext(1); + context.Log += (sender, e) => Console.WriteLine(e); //MYTODO: added for debugging purposes var dataPath = DatasetUtil.GetMlNetGeneratedRegressionDataset(); var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.MlNetGeneratedRegressionLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); @@ -143,8 +146,14 @@ public void AutoFitRegressionTest(string culture) .Execute(trainData, validationData, new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); + //var trainers = Enum.GetValues(typeof(RegressionTrainer)).OfType().ToList(); + //MYTODO: Only adding this for debugging purposes on the CI: - System.Console.WriteLine($"culture:{culture} - Count: {result.RunDetails.Count()} - Null ValidationMetrics Count:{result.RunDetails.Where(rd => rd.ValidationMetrics == null).Count()}"); + var nullValidationMetrics = result.RunDetails.Where(rd => rd.ValidationMetrics == null).ToArray(); + System.Console.WriteLine($"culture:{culture} - Count: {result.RunDetails.Count()} - Null ValidationMetrics Count:{nullValidationMetrics.Count()}"); + if (nullValidationMetrics.Count() > 0) + for (var i = 0; i < nullValidationMetrics.Count(); i++) + Console.WriteLine($"nullValidationMetrics[{i}].TrainerName={nullValidationMetrics[i].TrainerName}"); Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); From c8eb41dbcd121bcc7cfd2281ce58778bc6931b89 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 29 Oct 2020 14:20:22 -0700 Subject: [PATCH 09/13] Revert "Debug runSpecific" This reverts commit 95b728099415cacbe8cf3819ec51ce50cec94eb2. --- .vsts-dotnet-ci.yml | 14 +++++++------- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 13 ++----------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 9046e50b1d..5c2dcf30cb 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,7 +27,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - runSpecific: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -36,7 +36,7 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - runSpecific: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -44,7 +44,7 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - runSpecific: true + innerLoop: true pool: name: Hosted macOS @@ -63,7 +63,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -72,7 +72,7 @@ jobs: parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -92,7 +92,7 @@ jobs: _config_short: RFX _includeBenchmarkData: false _targetFramework: win-x64 - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -102,7 +102,7 @@ jobs: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 9ebee2f7df..f2c23f679f 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Globalization; using System.Linq; using System.Threading; @@ -106,7 +105,6 @@ private void Context_Log(object sender, LoggingEventArgs e) } [Theory] - [Trait("Category", "RunSpecificTest")] [InlineData("en-US")] [InlineData("ar-SA")] [InlineData("pl-PL")] @@ -115,7 +113,7 @@ public void AutoFitRegressionTest(string culture) var originalCulture = Thread.CurrentThread.CurrentCulture; Thread.CurrentThread.CurrentCulture = new CultureInfo(culture); - uint experimentTime = 30; + uint experimentTime = 0; if (culture == "ar-SA") { @@ -134,7 +132,6 @@ public void AutoFitRegressionTest(string culture) } var context = new MLContext(1); - context.Log += (sender, e) => Console.WriteLine(e); //MYTODO: added for debugging purposes var dataPath = DatasetUtil.GetMlNetGeneratedRegressionDataset(); var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.MlNetGeneratedRegressionLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); @@ -146,14 +143,8 @@ public void AutoFitRegressionTest(string culture) .Execute(trainData, validationData, new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); - //var trainers = Enum.GetValues(typeof(RegressionTrainer)).OfType().ToList(); - //MYTODO: Only adding this for debugging purposes on the CI: - var nullValidationMetrics = result.RunDetails.Where(rd => rd.ValidationMetrics == null).ToArray(); - System.Console.WriteLine($"culture:{culture} - Count: {result.RunDetails.Count()} - Null ValidationMetrics Count:{nullValidationMetrics.Count()}"); - if (nullValidationMetrics.Count() > 0) - for (var i = 0; i < nullValidationMetrics.Count(); i++) - Console.WriteLine($"nullValidationMetrics[{i}].TrainerName={nullValidationMetrics[i].TrainerName}"); + System.Console.WriteLine($"culture:{culture} - Count: {result.RunDetails.Count()} - Null ValidationMetrics Count:{result.RunDetails.Where(rd => rd.ValidationMetrics == null).Count()}"); Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); From 3f42e0773d341fdfbb0cbb289dea6a607cf0a7f0 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 29 Oct 2020 15:02:23 -0700 Subject: [PATCH 10/13] Removed LightGBM and addressed comments --- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index f2c23f679f..7b5de5e05e 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Globalization; using System.Linq; using System.Threading; @@ -111,44 +112,48 @@ private void Context_Log(object sender, LoggingEventArgs e) public void AutoFitRegressionTest(string culture) { var originalCulture = Thread.CurrentThread.CurrentCulture; - Thread.CurrentThread.CurrentCulture = new CultureInfo(culture); - - uint experimentTime = 0; - - if (culture == "ar-SA") + try { - // If users run AutoML with a different local, sometimes + Thread.CurrentThread.CurrentCulture = new CultureInfo(culture); + + // If users run AutoML with a different locale, sometimes // the sweeper encounters problems when parsing some strings. // So testing in another culture is necessary. - // Furthermore, these issues might only occur after several + // Furthermore, these issues might only occur after ~70 // iterations, so more experiment time is needed for this to // occur. - experimentTime = 30; + uint experimentTime = (uint) (culture == "en-US" ? 0 : 30); - } - else if(culture == "pl-PL") - { - experimentTime = 100; - } + var experimentSettings = new RegressionExperimentSettings { MaxExperimentTimeInSeconds = experimentTime}; + if (!Environment.Is64BitProcess) + { + // LightGBM isn't available on x86 machines + experimentSettings.Trainers.Remove(RegressionTrainer.LightGbm); + } - var context = new MLContext(1); - var dataPath = DatasetUtil.GetMlNetGeneratedRegressionDataset(); - var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.MlNetGeneratedRegressionLabel); - var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); - var trainData = textLoader.Load(dataPath); - var validationData = context.Data.TakeRows(trainData, 20); - trainData = context.Data.SkipRows(trainData, 20); - var result = context.Auto() - .CreateRegressionExperiment(experimentTime) - .Execute(trainData, validationData, - new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); + var context = new MLContext(1); + var dataPath = DatasetUtil.GetMlNetGeneratedRegressionDataset(); + var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.MlNetGeneratedRegressionLabel); + var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); + var trainData = textLoader.Load(dataPath); + var validationData = context.Data.TakeRows(trainData, 20); + trainData = context.Data.SkipRows(trainData, 20); + var result = context.Auto() + .CreateRegressionExperiment(experimentSettings) + .Execute(trainData, validationData, + new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); - //MYTODO: Only adding this for debugging purposes on the CI: - System.Console.WriteLine($"culture:{culture} - Count: {result.RunDetails.Count()} - Null ValidationMetrics Count:{result.RunDetails.Where(rd => rd.ValidationMetrics == null).Count()}"); + Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); - Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); + // Ensure experimentTime allows enough iterations to fully test the internationalization code + // If the below assertion fails, increase the experiment time so the number of iterations is met + Assert.True(culture == "en-US" || result.RunDetails.Count() >= 75, $"RunDetails.Count() = {result.RunDetails.Count()}, below 75"); - Thread.CurrentThread.CurrentCulture = originalCulture; + } + finally + { + Thread.CurrentThread.CurrentCulture = originalCulture; + } } [LightGBMFact] From d41623a0113a1d3c5fa29e438a920be1f635e1be Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 29 Oct 2020 15:03:13 -0700 Subject: [PATCH 11/13] Increased time --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 7b5de5e05e..cf7b92f514 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -122,7 +122,7 @@ public void AutoFitRegressionTest(string culture) // Furthermore, these issues might only occur after ~70 // iterations, so more experiment time is needed for this to // occur. - uint experimentTime = (uint) (culture == "en-US" ? 0 : 30); + uint experimentTime = (uint) (culture == "en-US" ? 0 : 65); var experimentSettings = new RegressionExperimentSettings { MaxExperimentTimeInSeconds = experimentTime}; if (!Environment.Is64BitProcess) From 645f590cf82765507230d060eb54d357d9b87cc1 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 29 Oct 2020 16:58:59 -0700 Subject: [PATCH 12/13] Increase time --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index cf7b92f514..d0ad42ad4b 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -122,7 +122,7 @@ public void AutoFitRegressionTest(string culture) // Furthermore, these issues might only occur after ~70 // iterations, so more experiment time is needed for this to // occur. - uint experimentTime = (uint) (culture == "en-US" ? 0 : 65); + uint experimentTime = (uint) (culture == "en-US" ? 0 : 100); var experimentSettings = new RegressionExperimentSettings { MaxExperimentTimeInSeconds = experimentTime}; if (!Environment.Is64BitProcess) From e37266442984e5f39205f09c0481ef6aee2bf074 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 29 Oct 2020 18:14:49 -0700 Subject: [PATCH 13/13] Increased time --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index d0ad42ad4b..bea3f97f3f 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -122,7 +122,7 @@ public void AutoFitRegressionTest(string culture) // Furthermore, these issues might only occur after ~70 // iterations, so more experiment time is needed for this to // occur. - uint experimentTime = (uint) (culture == "en-US" ? 0 : 100); + uint experimentTime = (uint) (culture == "en-US" ? 0 : 180); var experimentSettings = new RegressionExperimentSettings { MaxExperimentTimeInSeconds = experimentTime}; if (!Environment.Is64BitProcess)