From 477e2be96b88b83d628aa77eab08338165911db0 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 4 Mar 2020 13:48:28 -0800 Subject: [PATCH 1/5] try lightgbm tests with threads number to number of processor --- .../LightGbmTrainerBase.cs | 2 ++ .../TensorflowTests.cs | 1 - .../TrainerEstimators/TreeEstimators.cs | 17 +++++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs index 4fef5699c0..cb48c38b43 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs @@ -262,6 +262,8 @@ internal virtual Dictionary ToDictionary(IHost host) res["verbose"] = Silent ? "-1" : "1"; if (NumberOfThreads.HasValue) res["nthread"] = NumberOfThreads.Value; + else + res["nthread"] = Environment.ProcessorCount; res["seed"] = (Seed.HasValue) ? Seed : host.Rand.Next(); diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 857e642be9..921c6b2fe5 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -816,7 +816,6 @@ private void ExecuteTFTransformMNISTConvTrainingTest(bool shuffle, int? shuffleS LabelColumnName = "Label", FeatureColumnName = "Features", Seed = 1, - NumberOfThreads = 1, NumberOfIterations = 1 })); diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index c119e33c99..5799c86358 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -55,7 +55,6 @@ public void LightGBMBinaryEstimator() var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, - NumberOfThreads = 1, MinimumExampleCountPerLeaf = 2, UnbalancedSets = false, // default value }); @@ -76,7 +75,6 @@ public void LightGBMBinaryEstimatorUnbalanced() var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, - NumberOfThreads = 1, MinimumExampleCountPerLeaf = 2, UnbalancedSets = true, }); @@ -101,7 +99,6 @@ public void LightGBMBinaryEstimatorCorrectSigmoid() var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, - NumberOfThreads = 1, MinimumExampleCountPerLeaf = 2, Sigmoid = sigmoid }); @@ -220,7 +217,6 @@ public void LightGBMRegressorEstimator() var dataView = GetRegressionPipeline(); var trainer = ML.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options { - NumberOfThreads = 1, NormalizeFeatures = NormalizeOption.Warn, L2CategoricalRegularization = 5, }); @@ -931,7 +927,11 @@ public void LightGbmRegressorTestSummary() { var dataView = GetRegressionPipeline(); var trainer = ML.Regression.Trainers.LightGbm( - new LightGbmRegressionTrainer.Options { NumberOfIterations = 10, NumberOfThreads = 1, NumberOfLeaves = 5}); + new LightGbmRegressionTrainer.Options + { + NumberOfIterations = 10, + NumberOfLeaves = 5 + }); var transformer = trainer.Fit(dataView); @@ -985,7 +985,12 @@ public void LightGbmBinaryClassificationTestSummary() { var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); var trainer = pipeline.Append(ML.BinaryClassification.Trainers.LightGbm( - new LightGbmBinaryTrainer.Options { NumberOfIterations = 10, NumberOfThreads = 1, NumberOfLeaves = 5, UseCategoricalSplit = true })); + new LightGbmBinaryTrainer.Options + { + NumberOfIterations = 10, + NumberOfLeaves = 5, + UseCategoricalSplit = true + })); var transformer = trainer.Fit(dataView); From cb18de3cee62cfd484c94067ae3288ad6932bb62 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 4 Mar 2020 13:49:39 -0800 Subject: [PATCH 2/5] enable test run parallelization for ML.Test assembly --- test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj | 4 ++++ test/Microsoft.ML.Tests/Properties/AssemblyInfo.cs | 9 --------- 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 test/Microsoft.ML.Tests/Properties/AssemblyInfo.cs diff --git a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj index 0e0414cf5c..cff0b66736 100644 --- a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj +++ b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj @@ -64,4 +64,8 @@ PreserveNewest + + + + diff --git a/test/Microsoft.ML.Tests/Properties/AssemblyInfo.cs b/test/Microsoft.ML.Tests/Properties/AssemblyInfo.cs deleted file mode 100644 index c8afa8dcba..0000000000 --- a/test/Microsoft.ML.Tests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Xunit; - -// TODO: [TEST_STABILITY] disable test parallelization for this assembly as running test in parallel sometimes cause test host process to crash -[assembly: CollectionBehavior(DisableTestParallelization = true)] - From 23b0d7b20fb46fc2c73774fc36ae95aa7051b966 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 4 Mar 2020 14:37:29 -0800 Subject: [PATCH 3/5] try test on private agent pool see if issue can be repro --- .vsts-dotnet-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 3ae84667ed..70d8f1f0b6 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -65,7 +65,7 @@ jobs: _targetFramework: netcoreapp3.0 innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test - template: /build/ci/job-template.yml parameters: @@ -73,7 +73,7 @@ jobs: buildScript: build.cmd innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test - template: /build/ci/job-template.yml parameters: @@ -92,7 +92,7 @@ jobs: _targetFramework: win-x64 innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test - template: /build/ci/job-template.yml parameters: @@ -101,4 +101,4 @@ jobs: buildScript: build.cmd innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test From cf16c7d84acc1cfa664e79e95876658ca3d3f3cd Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 4 Mar 2020 22:00:37 -0800 Subject: [PATCH 4/5] revert unnecessary changes --- .vsts-dotnet-ci.yml | 8 ++++---- src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 70d8f1f0b6..3ae84667ed 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -65,7 +65,7 @@ jobs: _targetFramework: netcoreapp3.0 innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 - template: /build/ci/job-template.yml parameters: @@ -73,7 +73,7 @@ jobs: buildScript: build.cmd innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 - template: /build/ci/job-template.yml parameters: @@ -92,7 +92,7 @@ jobs: _targetFramework: win-x64 innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 - template: /build/ci/job-template.yml parameters: @@ -101,4 +101,4 @@ jobs: buildScript: build.cmd innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 diff --git a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs index cb48c38b43..4fef5699c0 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs @@ -262,8 +262,6 @@ internal virtual Dictionary ToDictionary(IHost host) res["verbose"] = Silent ? "-1" : "1"; if (NumberOfThreads.HasValue) res["nthread"] = NumberOfThreads.Value; - else - res["nthread"] = Environment.ProcessorCount; res["seed"] = (Seed.HasValue) ? Seed : host.Rand.Next(); From 81992449b2204d7af052d2ff8d2e75d42071377c Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Thu, 5 Mar 2020 11:36:33 -0800 Subject: [PATCH 5/5] add comments to explain the issue --- .../TensorflowTests.cs | 2 ++ .../TrainerEstimators/TreeEstimators.cs | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 921c6b2fe5..6a1b0cba6c 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -811,6 +811,8 @@ private void ExecuteTFTransformMNISTConvTrainingTest(bool shuffle, int? shuffleS batchSize: 20)) .Append(mlContext.Transforms.Concatenate("Features", "Prediction")) .AppendCacheCheckpoint(mlContext) + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 .Append(mlContext.MulticlassClassification.Trainers.LightGbm(new Trainers.LightGbm.LightGbmMulticlassTrainer.Options() { LabelColumnName = "Label", diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 5799c86358..e503133e91 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -52,6 +52,8 @@ public void LightGBMBinaryEstimator() { var (pipe, dataView) = GetBinaryClassificationPipeline(); + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, @@ -72,6 +74,8 @@ public void LightGBMBinaryEstimatorUnbalanced() { var (pipe, dataView) = GetBinaryClassificationPipeline(); + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, @@ -96,6 +100,8 @@ public void LightGBMBinaryEstimatorCorrectSigmoid() var (pipe, dataView) = GetBinaryClassificationPipeline(); var sigmoid = .789; + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, @@ -215,6 +221,9 @@ public void FastTreeRegressorEstimator() public void LightGBMRegressorEstimator() { var dataView = GetRegressionPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 var trainer = ML.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options { NormalizeFeatures = NormalizeOption.Warn, @@ -926,6 +935,9 @@ public void FastTreeTweedieRegressorTestSummary() public void LightGbmRegressorTestSummary() { var dataView = GetRegressionPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 var trainer = ML.Regression.Trainers.LightGbm( new LightGbmRegressionTrainer.Options { @@ -984,6 +996,9 @@ public void FastForestBinaryClassificationTestSummary() public void LightGbmBinaryClassificationTestSummary() { var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 var trainer = pipeline.Append(ML.BinaryClassification.Trainers.LightGbm( new LightGbmBinaryTrainer.Options {