From 8d0e1939d748ec99337ceef14d70045fceac5a52 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 13 Jun 2019 11:43:15 -0700 Subject: [PATCH 1/4] Temporarily remove the dataframe examples from the test run to see how much that effects the test length. --- src/python/tests/test_docs_example.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 50333cd9..89e6ec1d 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -33,12 +33,12 @@ def test_examples(self): "Unable to find examples in '{0}'".format(fold)) # also include the 'examples_from_dataframe' files - fold_df = os.path.join(fold, 'examples_from_dataframe') - fold_files_df = [(fold_df, _) for _ in os.listdir( - fold_df) if os.path.splitext(_)[-1] == '.py'] - - # merge details of all examples into one list - fold_files.extend(fold_files_df) +# fold_df = os.path.join(fold, 'examples_from_dataframe') +# fold_files_df = [(fold_df, _) for _ in os.listdir( +# fold_df) if os.path.splitext(_)[-1] == '.py'] +# +# # merge details of all examples into one list +# fold_files.extend(fold_files_df) fold_files.sort() modpath = os.path.abspath(os.path.dirname(myfile)) From 66f8ffc221a7f1862f8c33701c82dfb503ebc432 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 13 Jun 2019 12:55:17 -0700 Subject: [PATCH 2/4] Remove all examples from the tests to see how it impacts the CI run. --- src/python/tests/test_docs_example.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/python/tests/test_docs_example.py b/src/python/tests/test_docs_example.py index 89e6ec1d..5fe2d848 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests/test_docs_example.py @@ -15,6 +15,7 @@ class TestDocsExamples(unittest.TestCase): + @unittest.skip def test_examples(self): this = os.path.abspath(os.path.dirname(__file__)) fold = os.path.normpath( @@ -33,12 +34,12 @@ def test_examples(self): "Unable to find examples in '{0}'".format(fold)) # also include the 'examples_from_dataframe' files -# fold_df = os.path.join(fold, 'examples_from_dataframe') -# fold_files_df = [(fold_df, _) for _ in os.listdir( -# fold_df) if os.path.splitext(_)[-1] == '.py'] -# -# # merge details of all examples into one list -# fold_files.extend(fold_files_df) + fold_df = os.path.join(fold, 'examples_from_dataframe') + fold_files_df = [(fold_df, _) for _ in os.listdir( + fold_df) if os.path.splitext(_)[-1] == '.py'] + + # merge details of all examples into one list + fold_files.extend(fold_files_df) fold_files.sort() modpath = os.path.abspath(os.path.dirname(myfile)) From f5c5337ab8a5cb9185278486baf6c10d2068b89a Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 13 Jun 2019 15:39:59 -0700 Subject: [PATCH 3/4] Put long running tests in to their own folder to shorten build times. --- build.cmd | 16 +++++++++++++++- build.sh | 13 ++++++++++++- build/ci/phase-template.yml | 12 +++++++----- build/vsts-ci-nightly.yml | 9 ++++++++- .../test_docs_example.py | 1 - .../test_docs_notebooks.py | 0 6 files changed, 42 insertions(+), 9 deletions(-) rename src/python/{tests => tests_extended}/test_docs_example.py (99%) rename src/python/{tests => tests_extended}/test_docs_notebooks.py (100%) diff --git a/build.cmd b/build.cmd index 8669ceff..c443fee6 100644 --- a/build.cmd +++ b/build.cmd @@ -21,6 +21,7 @@ set BoostRoot=%DependenciesDir%BoostDbg3.7 set PythonVersion=3.7 set PythonTag=cp37 set RunTests=False +set RunExtendedTests=False set BuildDotNetBridgeOnly=False set SkipDotNetBridge=False @@ -33,6 +34,10 @@ if /i [%1] == [--runTests] ( set RunTests=True shift && goto :Arg_Loop ) +if /i [%1] == [--includeExtendedTests] ( + set RunExtendedTests=True + shift && goto :Arg_Loop +) if /i [%1] == [--buildDotNetBridgeOnly] ( set BuildDotNetBridgeOnly=True shift && goto :Arg_Loop @@ -43,11 +48,12 @@ if /i [%1] == [--skipDotNetBridge] ( ) else goto :Usage :Usage -echo "Usage: build.cmd [--configuration ] [--runTests] [--buildDotNetBridgeOnly] [--skipDotNetBridge]" +echo "Usage: build.cmd [--configuration ] [--runTests] [--includeExtendedTests] [--buildDotNetBridgeOnly] [--skipDotNetBridge]" echo "" echo "Options:" echo " --configuration Build Configuration (DbgWinPy3.7,DbgWinPy3.6,DbgWinPy3.5,DbgWinPy2.7,RlsWinPy3.7,RlsWinPy3.6,RlsWinPy3.5,RlsWinPy2.7)" echo " --runTests Run tests after build" +echo " --includeExtendedTests Include the extended tests if the tests are run" echo " --buildDotNetBridgeOnly Build only DotNetBridge" echo " --skipDotNetBridge Build everything except DotNetBridge" goto :Exit_Success @@ -326,6 +332,7 @@ call "%PythonExe%" -m pip install "scikit-learn==0.19.2" set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests +set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( @@ -336,6 +343,13 @@ if errorlevel 1 ( goto :Exit_Error ) +if "%RunExtendedTests%" == "True" ( + call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" + if errorlevel 1 ( + goto :Exit_Error + ) +) + :Exit_Success endlocal exit /b %ERRORLEVEL% diff --git a/build.sh b/build.sh index d6c77c5e..78de9ff8 100755 --- a/build.sh +++ b/build.sh @@ -11,12 +11,13 @@ mkdir -p "${DependenciesDir}" usage() { - echo "Usage: $0 --configuration [--runTests]" + echo "Usage: $0 --configuration [--runTests] [--includeExtendedTests]" echo "" echo "Options:" echo " --configuration Build Configuration (DbgLinPy3.7,DbgLinPy3.6,DbgLinPy3.5,DbgLinPy2.7,RlsLinPy3.7,RlsLinPy3.6,RlsLinPy3.5,RlsLinPy2.7,DbgMacPy3.7,DbgMacPy3.6,DbgMacPy3.5,DbgMacPy2.7,RlsMacPy3.7,RlsMacPy3.6,RlsMacPy3.5,RlsMacPy2.7)" echo " --runTests Run tests after build" echo " --runTestsOnly Run tests on a wheel file in default build location (/target/)" + echo " --includeExtendedTests Include the extended tests if the tests are run" echo " --buildNativeBridgeOnly Build only the native bridge code" echo " --skipNativeBridge Build the DotNet bridge and python wheel but use existing native bridge binaries (e.g. /x64/DbgLinPy3.7/pybridge.so)" exit 1 @@ -30,6 +31,7 @@ else __configuration=DbgLinPy3.7 fi __runTests=false +__runExtendedTests=false __buildNativeBridge=true __buildDotNetBridge=true @@ -47,6 +49,9 @@ while [ "$1" != "" ]; do --runtests) __runTests=true ;; + --includeextendedtests) + __runExtendedTests=true + ;; --runtestsonly) __buildNativeBridge=false __buildDotNetBridge=false @@ -268,9 +273,15 @@ then PackagePath=${PythonRoot}/lib/python${PythonVersion}/site-packages/nimbusml TestsPath1=${PackagePath}/tests TestsPath2=${__currentScriptDir}/src/python/tests + TestsPath3=${__currentScriptDir}/src/python/tests_extended ReportPath=${__currentScriptDir}/build/TestCoverageReport "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}" "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}" + + if [ ${__runExtendedTests} = true ] + then + "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath3}" + fi fi exit $? diff --git a/build/ci/phase-template.yml b/build/ci/phase-template.yml index ce357221..4df9692c 100644 --- a/build/ci/phase-template.yml +++ b/build/ci/phase-template.yml @@ -4,6 +4,7 @@ parameters: buildMatrix: {} buildQueue: {} testDistro: '' + testOptions: '' phases: @@ -12,6 +13,7 @@ phases: _buildScript: ${{ parameters.buildScript }} _dockerRun: docker run -e SYSTEM_TEAMFOUNDATIONCOLLECTIONURI="$(System.TeamFoundationCollectionUri)" -e BUILD_BUILDNUMBER="$(Build.BuildNumber)" -i -v $(Build.SourcesDirectory):/builddir -w="/builddir" _distro: ${{ parameters.testDistro }} + _testOptions: ${{ parameters.testOptions }} queue: parallel: 99 matrix: @@ -21,14 +23,14 @@ phases: # Windows phases - ${{ if eq(parameters.name, 'Windows') }}: - - script: $(_buildScript) --configuration $(_configuration) --runTests + - script: $(_buildScript) --configuration $(_configuration) --runTests $(_testOptions) # Mac phases - ${{ if eq(parameters.name, 'Mac') }}: - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb mono-libgdiplus gettext && brew link gettext --force - ${{ if eq(parameters.testDistro, 'noTests') }}: - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) - ${{ if eq(parameters.testDistro, '') }}: - - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTests + - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTests $(_testOptions) # Linux phases - ${{ if or(eq(parameters.name, 'Linux_Ubuntu16'), eq(parameters.name, 'Linux_Ubuntu14'), eq(parameters.name, 'Linux_CentOS7')) }}: - script: $(_dockerRun) mlnet/ubuntu16-nativebuild:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --buildNativeBridgeOnly" @@ -36,13 +38,13 @@ phases: - script: $(_dockerRun) mlnet/ubuntu16-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --skipNativeBridge" displayName: Build python wheel - ${{ if eq(parameters.testDistro, 'ubuntu16') }}: - - script: $(_dockerRun) mlnet/ubuntu16-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly" + - script: $(_dockerRun) mlnet/ubuntu16-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly $(_testOptions)" displayName: Run tests Ubuntu16 - ${{ if eq(parameters.testDistro, 'ubuntu14') }}: - - script: $(_dockerRun) mlnet/ubuntu14-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly" + - script: $(_dockerRun) mlnet/ubuntu14-general:0.1 bash -c "source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly $(_testOptions)" displayName: Run tests Ubuntu14 - ${{ if eq(parameters.testDistro, 'centos7') }}: - - script: $(_dockerRun) mlnet/centos7-general:0.1 bash -c "source /root/.bash_profile && source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly" + - script: $(_dockerRun) mlnet/centos7-general:0.1 bash -c "source /root/.bash_profile && source /etc/profile && chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration) --runTestsOnly $(_testOptions)" displayName: Run tests CentOS7 # Publish build artifacts diff --git a/build/vsts-ci-nightly.yml b/build/vsts-ci-nightly.yml index 6e678411..e63127fe 100644 --- a/build/vsts-ci-nightly.yml +++ b/build/vsts-ci-nightly.yml @@ -16,6 +16,7 @@ phases: _configuration: RlsWinPy2.7 buildQueue: name: Hosted VS2017 + testOptions: --includeExtendedTests # Build all configurations for Mac - template: /build/ci/phase-template.yml @@ -33,6 +34,7 @@ phases: _configuration: RlsMacPy2.7 buildQueue: name: Hosted macOS + testOptions: --includeExtendedTests # Build all configurations for Linux # Run tests on Ubuntu16 @@ -52,6 +54,8 @@ phases: _configuration: RlsLinPy2.7 buildQueue: name: Hosted Ubuntu 1604 + testOptions: --includeExtendedTests + # Run tests on Ubuntu14 - template: /build/ci/phase-template.yml parameters: @@ -69,6 +73,8 @@ phases: _configuration: RlsLinPy2.7 buildQueue: name: Hosted Ubuntu 1604 + testOptions: --includeExtendedTests + # Run tests on CentOS7 - template: /build/ci/phase-template.yml parameters: @@ -85,4 +91,5 @@ phases: Py27: _configuration: RlsLinPy2.7 buildQueue: - name: Hosted Ubuntu 1604 \ No newline at end of file + name: Hosted Ubuntu 1604 + testOptions: --includeExtendedTests diff --git a/src/python/tests/test_docs_example.py b/src/python/tests_extended/test_docs_example.py similarity index 99% rename from src/python/tests/test_docs_example.py rename to src/python/tests_extended/test_docs_example.py index 5fe2d848..50333cd9 100644 --- a/src/python/tests/test_docs_example.py +++ b/src/python/tests_extended/test_docs_example.py @@ -15,7 +15,6 @@ class TestDocsExamples(unittest.TestCase): - @unittest.skip def test_examples(self): this = os.path.abspath(os.path.dirname(__file__)) fold = os.path.normpath( diff --git a/src/python/tests/test_docs_notebooks.py b/src/python/tests_extended/test_docs_notebooks.py similarity index 100% rename from src/python/tests/test_docs_notebooks.py rename to src/python/tests_extended/test_docs_notebooks.py From a0ecc89aa5a8ff9a0f43e48ba1de97d95c17c2e9 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 13 Jun 2019 16:07:14 -0700 Subject: [PATCH 4/4] Update nimbusml.pyproj to reflect the newly moved test files. Forgot to save the nimbusml.pyproj in visual studio. --- src/python/nimbusml.pyproj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 23bcd324..2fecda2d 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -646,9 +646,9 @@ - + - + @@ -700,6 +700,7 @@ +