From 8162567b3f464068091db5100e6b55711f422e80 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 14 Oct 2019 20:03:40 -0700 Subject: [PATCH 01/40] Execute the first two sets of unit tests in parallel. --- build.cmd | 6 +++--- build.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build.cmd b/build.cmd index 8ed5005d..84eab4dd 100644 --- a/build.cmd +++ b/build.cmd @@ -349,7 +349,7 @@ if "%InstallPythonPackages%" == "True" ( echo "Installing python packages ... " echo "#################################" call "%PythonExe%" -m pip install --upgrade pip - call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" + call "%PythonExe%" -m pip install --upgrade nose pytest pytest-xdist graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq @@ -375,11 +375,11 @@ set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport -call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) -call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath2%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath2%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) diff --git a/build.sh b/build.sh index 6d5221c9..b285e8d4 100755 --- a/build.sh +++ b/build.sh @@ -280,7 +280,7 @@ then exit 1 fi # Review: Adding "--upgrade" to pip install will cause problems when using Anaconda as the python distro because of Anaconda's quirks with pytest. - "${PythonExe}" -m pip install nose "pytest>=4.4.0" graphviz "pytest-cov>=2.6.1" "jupyter_client>=4.4.0" "nbconvert>=4.2.0" + "${PythonExe}" -m pip install nose "pytest>=4.4.0" pytest-xdist graphviz "pytest-cov>=2.6.1" "jupyter_client>=4.4.0" "nbconvert>=4.2.0" if [ ${PythonVersion} = 2.7 ] then "${PythonExe}" -m pip install --upgrade pyzmq @@ -307,8 +307,8 @@ then TestsPath2=${__currentScriptDir}/src/python/tests TestsPath3=${__currentScriptDir}/src/python/tests_extended ReportPath=${__currentScriptDir}/build/TestCoverageReport - "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}" - "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}" + "${PythonExe}" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "${TestsPath1}" + "${PythonExe}" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "${TestsPath2}" if [ ${__runExtendedTests} = true ] then From 1ac5033f831ea6368d65c25c518fc54fdfce30de Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 10:26:59 -0700 Subject: [PATCH 02/40] Wrap test estimator checks in a python unit test. --- src/python/tests/test_estimator_checks.py | 152 +++++++++++----------- 1 file changed, 78 insertions(+), 74 deletions(-) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index e40af618..1abcf050 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -7,6 +7,7 @@ """ import json import os +import unittest from nimbusml.decomposition import FactorizationMachineBinaryClassifier from nimbusml.ensemble import EnsembleClassifier @@ -272,88 +273,91 @@ def load_json(file_path): 'DatasetTransformer' ]) -epoints = [] -my_path = os.path.realpath(__file__) -my_dir = os.path.dirname(my_path) -manifest_diff_json = os.path.join(my_dir, '..', 'tools', - 'manifest_diff.json') -manifest_diff = load_json(manifest_diff_json) -for e in manifest_diff['EntryPoints']: - if e['NewName'] not in skip_epoints: - epoints.append((e['Module'], e['NewName'])) +class TestEstimatorChecks(unittest.TestCase): -all_checks = {} -all_failed_checks = {} -all_passed_checks = {} -total_checks_passed = 0 + def test_estimator_checks(self): + epoints = [] + my_path = os.path.realpath(__file__) + my_dir = os.path.dirname(my_path) + manifest_diff_json = os.path.join(my_dir, '..', 'tools', + 'manifest_diff.json') + manifest_diff = load_json(manifest_diff_json) + for e in manifest_diff['EntryPoints']: + if e['NewName'] not in skip_epoints: + epoints.append((e['Module'], e['NewName'])) -print("total entrypoints: {}", len(epoints)) + all_checks = {} + all_failed_checks = {} + all_passed_checks = {} + total_checks_passed = 0 -for e in epoints: - checks = set() - failed_checks = set() - passed_checks = set() - class_name = e[1] - print("======== now Estimator is %s =========== " % class_name) - # skip LighGbm for now, because of random crashes. - if 'LightGbm' in class_name: - continue + print("total entrypoints: {}", len(epoints)) - mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)]) - the_class = getattr(mod, class_name) - if class_name in INSTANCES: - estimator = INSTANCES[class_name] - else: - estimator = the_class() + for e in epoints: + checks = set() + failed_checks = set() + passed_checks = set() + class_name = e[1] + print("======== now Estimator is %s =========== " % class_name) + # skip LighGbm for now, because of random crashes. + if 'LightGbm' in class_name: + continue - if estimator._use_single_input_as_string(): - estimator = estimator << 'F0' + mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)]) + the_class = getattr(mod, class_name) + if class_name in INSTANCES: + estimator = INSTANCES[class_name] + else: + estimator = the_class() - for check in _yield_all_checks(class_name, estimator): - # Skip check_dict_unchanged for estimators which - # update the classes_ attribute. For more details - # see https://github.com/microsoft/NimbusML/pull/200 - if (check.__name__ == 'check_dict_unchanged') and \ - (hasattr(estimator, 'predict_proba') or - hasattr(estimator, 'decision_function')): - continue + if estimator._use_single_input_as_string(): + estimator = estimator << 'F0' - if check.__name__ in OMITTED_CHECKS_ALWAYS: - continue - if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS: - continue - if class_name in OMITTED_CHECKS and check.__name__ in \ - OMITTED_CHECKS[class_name]: - continue - if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \ - OMITTED_CHECKS_TUPLE[1]: - continue - checks.add(check.__name__) - try: - check(class_name, estimator.clone()) - passed_checks.add(check.__name__) - total_checks_passed = total_checks_passed + 1 - except Exception as e: - failed_checks.add(check.__name__) + for check in _yield_all_checks(class_name, estimator): + # Skip check_dict_unchanged for estimators which + # update the classes_ attribute. For more details + # see https://github.com/microsoft/NimbusML/pull/200 + if (check.__name__ == 'check_dict_unchanged') and \ + (hasattr(estimator, 'predict_proba') or + hasattr(estimator, 'decision_function')): + continue - if frozenset(checks) not in all_checks: - all_checks[frozenset(checks)] = [] - all_checks[frozenset(checks)].append(class_name) + if check.__name__ in OMITTED_CHECKS_ALWAYS: + continue + if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS: + continue + if class_name in OMITTED_CHECKS and check.__name__ in \ + OMITTED_CHECKS[class_name]: + continue + if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \ + OMITTED_CHECKS_TUPLE[1]: + continue + checks.add(check.__name__) + try: + check(class_name, estimator.clone()) + passed_checks.add(check.__name__) + total_checks_passed = total_checks_passed + 1 + except Exception as e: + failed_checks.add(check.__name__) - if len(failed_checks) > 0: - if frozenset(failed_checks) not in all_failed_checks: - all_failed_checks[frozenset(failed_checks)] = [] - all_failed_checks[frozenset(failed_checks)].append(class_name) + if frozenset(checks) not in all_checks: + all_checks[frozenset(checks)] = [] + all_checks[frozenset(checks)].append(class_name) - if frozenset(passed_checks) not in all_passed_checks: - all_passed_checks[frozenset(passed_checks)] = [] - all_passed_checks[frozenset(passed_checks)].append(class_name) + if len(failed_checks) > 0: + if frozenset(failed_checks) not in all_failed_checks: + all_failed_checks[frozenset(failed_checks)] = [] + all_failed_checks[frozenset(failed_checks)].append(class_name) -if len(all_failed_checks) > 0: - print("Following tests failed for components:") - for key, value in all_failed_checks.items(): - print('========================') - print(key) - print(value) - raise RuntimeError("estimator checks failed") -print("success, total checks passed %s ", total_checks_passed) + if frozenset(passed_checks) not in all_passed_checks: + all_passed_checks[frozenset(passed_checks)] = [] + all_passed_checks[frozenset(passed_checks)].append(class_name) + + if len(all_failed_checks) > 0: + print("Following tests failed for components:") + for key, value in all_failed_checks.items(): + print('========================') + print(key) + print(value) + raise RuntimeError("estimator checks failed") + print("success, total checks passed %s ", total_checks_passed) From 04ea00f336b7406bab9f5596f2b79d80af2ffa3f Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 10:50:20 -0700 Subject: [PATCH 03/40] Combine the non-extended test runs together to make them more parallelizable. --- build.cmd | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build.cmd b/build.cmd index 84eab4dd..2ffb6ce4 100644 --- a/build.cmd +++ b/build.cmd @@ -375,11 +375,7 @@ set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport -call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" -if errorlevel 1 ( - goto :Exit_Error -) -call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath2%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath1%" "%TestsPath2%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From f25bb32d6f0e4cd4d65d8d58d50e002ae7d4c93f Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 11:06:27 -0700 Subject: [PATCH 04/40] Reverse the tests path args order to try and have test_estimator_checks run earlier in the test run. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index 2ffb6ce4..e994ab7c 100644 --- a/build.cmd +++ b/build.cmd @@ -375,7 +375,7 @@ set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport -call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath1%" "%TestsPath2%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From eeb4e3067260445de3b2fb9295087f4d145b77fa Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 11:08:52 -0700 Subject: [PATCH 05/40] Group the unit tests by filename to make their execution more stable. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index e994ab7c..619058f3 100644 --- a/build.cmd +++ b/build.cmd @@ -375,7 +375,7 @@ set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport -call "%PythonExe%" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n auto --dist=loadfile --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From a591122ec2d6081bbd2bd02b01128bd1b2c65f3e Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 11:15:20 -0700 Subject: [PATCH 06/40] Add the build script changes to build.sh --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index b285e8d4..76a3775c 100755 --- a/build.sh +++ b/build.sh @@ -307,8 +307,7 @@ then TestsPath2=${__currentScriptDir}/src/python/tests TestsPath3=${__currentScriptDir}/src/python/tests_extended ReportPath=${__currentScriptDir}/build/TestCoverageReport - "${PythonExe}" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "${TestsPath1}" - "${PythonExe}" -m pytest -n auto --verbose --maxfail=1000 --capture=sys "${TestsPath2}" + "${PythonExe}" -m pytest -n auto --dist=loadfile --verbose --maxfail=1000 --capture=sys "${TestsPath2}" "${TestsPath1}" if [ ${__runExtendedTests} = true ] then From 7e2a1607fdc58cdfe4915078cd541a6d7737a520 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 12:02:40 -0700 Subject: [PATCH 07/40] Set the minimum number of concurrent unit tests to 4 for the windows based test runs. --- build.cmd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index 619058f3..733d093e 100644 --- a/build.cmd +++ b/build.cmd @@ -375,7 +375,10 @@ set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport -call "%PythonExe%" -m pytest -n auto --dist=loadfile --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +set NumConcurrentTests=%NUMBER_OF_PROCESSORS% +if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4 + +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --dist=loadfile --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From e9557adc7cf8048a2363986303a182addf47213b Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 14:13:59 -0700 Subject: [PATCH 08/40] Split test_estimator_checks into two separate tests. --- build.cmd | 2 +- src/python/tests/test_estimator_checks.py | 197 ++++++++++++---------- 2 files changed, 106 insertions(+), 93 deletions(-) diff --git a/build.cmd b/build.cmd index 733d093e..55ccab28 100644 --- a/build.cmd +++ b/build.cmd @@ -378,7 +378,7 @@ set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4 -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --dist=loadfile --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index 1abcf050..472fced4 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -248,6 +248,14 @@ MULTI_OUTPUT.extend(MULTI_OUTPUT_EX) +skip_epoints = set([ + 'OneVsRestClassifier', + 'TreeFeaturizer', + # skip SymSgdBinaryClassifier for now, because of crashes. + 'SymSgdBinaryClassifier', + 'DatasetTransformer' +]) + def my_import(name): components = name.split('.') @@ -264,100 +272,105 @@ def load_json(file_path): content_without_comments = '\n'.join(lines) return json.loads(content_without_comments) +def get_epoints(): + epoints = [] + my_path = os.path.realpath(__file__) + my_dir = os.path.dirname(my_path) + manifest_diff_json = os.path.join(my_dir, '..', 'tools', + 'manifest_diff.json') + manifest_diff = load_json(manifest_diff_json) + for e in manifest_diff['EntryPoints']: + if e['NewName'] not in skip_epoints: + epoints.append((e['Module'], e['NewName'])) -skip_epoints = set([ - 'OneVsRestClassifier', - 'TreeFeaturizer', - # skip SymSgdBinaryClassifier for now, because of crashes. - 'SymSgdBinaryClassifier', - 'DatasetTransformer' -]) + return epoints -class TestEstimatorChecks(unittest.TestCase): +def check_entry_points(epoints): + all_checks = {} + all_failed_checks = {} + all_passed_checks = {} + total_checks_passed = 0 + + print("total entrypoints: {}", len(epoints)) + + for e in epoints: + checks = set() + failed_checks = set() + passed_checks = set() + class_name = e[1] + print("======== now Estimator is %s =========== " % class_name) + # skip LighGbm for now, because of random crashes. + if 'LightGbm' in class_name: + continue - def test_estimator_checks(self): - epoints = [] - my_path = os.path.realpath(__file__) - my_dir = os.path.dirname(my_path) - manifest_diff_json = os.path.join(my_dir, '..', 'tools', - 'manifest_diff.json') - manifest_diff = load_json(manifest_diff_json) - for e in manifest_diff['EntryPoints']: - if e['NewName'] not in skip_epoints: - epoints.append((e['Module'], e['NewName'])) - - all_checks = {} - all_failed_checks = {} - all_passed_checks = {} - total_checks_passed = 0 - - print("total entrypoints: {}", len(epoints)) - - for e in epoints: - checks = set() - failed_checks = set() - passed_checks = set() - class_name = e[1] - print("======== now Estimator is %s =========== " % class_name) - # skip LighGbm for now, because of random crashes. - if 'LightGbm' in class_name: + mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)]) + the_class = getattr(mod, class_name) + if class_name in INSTANCES: + estimator = INSTANCES[class_name] + else: + estimator = the_class() + + if estimator._use_single_input_as_string(): + estimator = estimator << 'F0' + + for check in _yield_all_checks(class_name, estimator): + # Skip check_dict_unchanged for estimators which + # update the classes_ attribute. For more details + # see https://github.com/microsoft/NimbusML/pull/200 + if (check.__name__ == 'check_dict_unchanged') and \ + (hasattr(estimator, 'predict_proba') or + hasattr(estimator, 'decision_function')): + continue + + if check.__name__ in OMITTED_CHECKS_ALWAYS: + continue + if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS: continue + if class_name in OMITTED_CHECKS and check.__name__ in \ + OMITTED_CHECKS[class_name]: + continue + if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \ + OMITTED_CHECKS_TUPLE[1]: + continue + checks.add(check.__name__) + try: + check(class_name, estimator.clone()) + passed_checks.add(check.__name__) + total_checks_passed = total_checks_passed + 1 + except Exception as e: + failed_checks.add(check.__name__) + + if frozenset(checks) not in all_checks: + all_checks[frozenset(checks)] = [] + all_checks[frozenset(checks)].append(class_name) + + if len(failed_checks) > 0: + if frozenset(failed_checks) not in all_failed_checks: + all_failed_checks[frozenset(failed_checks)] = [] + all_failed_checks[frozenset(failed_checks)].append(class_name) + + if frozenset(passed_checks) not in all_passed_checks: + all_passed_checks[frozenset(passed_checks)] = [] + all_passed_checks[frozenset(passed_checks)].append(class_name) + + if len(all_failed_checks) > 0: + print("Following tests failed for components:") + for key, value in all_failed_checks.items(): + print('========================') + print(key) + print(value) + raise RuntimeError("estimator checks failed") + print("success, total checks passed %s ", total_checks_passed) + + +class TestEstimatorChecks(unittest.TestCase): + + def test_estimator_checks_1(self): + epoints = get_epoints() + check_entry_points(epoints[:len(epoints)//2]) + + def test_estimator_checks_2(self): + epoints = get_epoints() + check_entry_points(epoints[len(epoints)//2:]) + - mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)]) - the_class = getattr(mod, class_name) - if class_name in INSTANCES: - estimator = INSTANCES[class_name] - else: - estimator = the_class() - - if estimator._use_single_input_as_string(): - estimator = estimator << 'F0' - - for check in _yield_all_checks(class_name, estimator): - # Skip check_dict_unchanged for estimators which - # update the classes_ attribute. For more details - # see https://github.com/microsoft/NimbusML/pull/200 - if (check.__name__ == 'check_dict_unchanged') and \ - (hasattr(estimator, 'predict_proba') or - hasattr(estimator, 'decision_function')): - continue - - if check.__name__ in OMITTED_CHECKS_ALWAYS: - continue - if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS: - continue - if class_name in OMITTED_CHECKS and check.__name__ in \ - OMITTED_CHECKS[class_name]: - continue - if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \ - OMITTED_CHECKS_TUPLE[1]: - continue - checks.add(check.__name__) - try: - check(class_name, estimator.clone()) - passed_checks.add(check.__name__) - total_checks_passed = total_checks_passed + 1 - except Exception as e: - failed_checks.add(check.__name__) - - if frozenset(checks) not in all_checks: - all_checks[frozenset(checks)] = [] - all_checks[frozenset(checks)].append(class_name) - - if len(failed_checks) > 0: - if frozenset(failed_checks) not in all_failed_checks: - all_failed_checks[frozenset(failed_checks)] = [] - all_failed_checks[frozenset(failed_checks)].append(class_name) - - if frozenset(passed_checks) not in all_passed_checks: - all_passed_checks[frozenset(passed_checks)] = [] - all_passed_checks[frozenset(passed_checks)].append(class_name) - - if len(all_failed_checks) > 0: - print("Following tests failed for components:") - for key, value in all_failed_checks.items(): - print('========================') - print(key) - print(value) - raise RuntimeError("estimator checks failed") - print("success, total checks passed %s ", total_checks_passed) From 9e6968c17ee1e3a97af570b2e3be45bb5612ff50 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 15 Oct 2019 14:38:51 -0700 Subject: [PATCH 09/40] Hard code num concurrent unit tests to 4 for linux and mac. Disable file based grouping of unit tests. --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 76a3775c..a78a8720 100755 --- a/build.sh +++ b/build.sh @@ -307,7 +307,7 @@ then TestsPath2=${__currentScriptDir}/src/python/tests TestsPath3=${__currentScriptDir}/src/python/tests_extended ReportPath=${__currentScriptDir}/build/TestCoverageReport - "${PythonExe}" -m pytest -n auto --dist=loadfile --verbose --maxfail=1000 --capture=sys "${TestsPath2}" "${TestsPath1}" + "${PythonExe}" -m pytest -n 4 --verbose --maxfail=1000 --capture=sys "${TestsPath2}" "${TestsPath1}" if [ ${__runExtendedTests} = true ] then From 265e04fd1199e2659080e425846e0bcabd3fce40 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Wed, 16 Oct 2019 09:15:40 -0700 Subject: [PATCH 10/40] Run the two extended tests in parallel. --- build.cmd | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cmd b/build.cmd index 36132e57..dfb40bf0 100644 --- a/build.cmd +++ b/build.cmd @@ -423,7 +423,7 @@ if errorlevel 1 ( ) if "%RunExtendedTests%" == "True" ( - call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" + call "%PythonExe%" -m pytest -n 2 --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) diff --git a/build.sh b/build.sh index a78a8720..ded01aa2 100755 --- a/build.sh +++ b/build.sh @@ -311,7 +311,7 @@ then if [ ${__runExtendedTests} = true ] then - "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath3}" + "${PythonExe}" -m pytest -n 2 --verbose --maxfail=1000 --capture=sys "${TestsPath3}" fi fi From 5df9ad356732c354e24d4067007127bc120abfe2 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Wed, 16 Oct 2019 14:18:22 -0700 Subject: [PATCH 11/40] Support running test_estimator_checks as the main file. --- src/python/tests/test_estimator_checks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index 472fced4..5249d9d3 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -374,3 +374,5 @@ def test_estimator_checks_2(self): check_entry_points(epoints[len(epoints)//2:]) +if __name__ == '__main__': + unittest.main() From 51cc91638cf37891e283be071ea072c9f5767c5d Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 17 Oct 2019 16:12:41 -0700 Subject: [PATCH 12/40] Dynamically generate the test_estimator_checks unit tests. --- src/python/tests/test_estimator_checks.py | 152 +++++++++------------- 1 file changed, 58 insertions(+), 94 deletions(-) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index 5249d9d3..accffb7f 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -257,14 +257,6 @@ ]) -def my_import(name): - components = name.split('.') - mod = __import__(components[0]) - for comp in components[1:]: - mod = getattr(mod, comp) - return mod - - def load_json(file_path): with open(file_path) as f: lines = f.readlines() @@ -280,98 +272,70 @@ def get_epoints(): 'manifest_diff.json') manifest_diff = load_json(manifest_diff_json) for e in manifest_diff['EntryPoints']: - if e['NewName'] not in skip_epoints: + if (e['NewName'] not in skip_epoints) and ('LightGbm' not in e['NewName']): epoints.append((e['Module'], e['NewName'])) return epoints -def check_entry_points(epoints): - all_checks = {} - all_failed_checks = {} - all_passed_checks = {} - total_checks_passed = 0 - - print("total entrypoints: {}", len(epoints)) - - for e in epoints: - checks = set() - failed_checks = set() - passed_checks = set() - class_name = e[1] - print("======== now Estimator is %s =========== " % class_name) - # skip LighGbm for now, because of random crashes. - if 'LightGbm' in class_name: - continue - - mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)]) - the_class = getattr(mod, class_name) - if class_name in INSTANCES: - estimator = INSTANCES[class_name] - else: - estimator = the_class() - - if estimator._use_single_input_as_string(): - estimator = estimator << 'F0' - - for check in _yield_all_checks(class_name, estimator): - # Skip check_dict_unchanged for estimators which - # update the classes_ attribute. For more details - # see https://github.com/microsoft/NimbusML/pull/200 - if (check.__name__ == 'check_dict_unchanged') and \ - (hasattr(estimator, 'predict_proba') or - hasattr(estimator, 'decision_function')): - continue - - if check.__name__ in OMITTED_CHECKS_ALWAYS: - continue - if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS: - continue - if class_name in OMITTED_CHECKS and check.__name__ in \ - OMITTED_CHECKS[class_name]: - continue - if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \ - OMITTED_CHECKS_TUPLE[1]: - continue - checks.add(check.__name__) - try: - check(class_name, estimator.clone()) - passed_checks.add(check.__name__) - total_checks_passed = total_checks_passed + 1 - except Exception as e: - failed_checks.add(check.__name__) - - if frozenset(checks) not in all_checks: - all_checks[frozenset(checks)] = [] - all_checks[frozenset(checks)].append(class_name) - - if len(failed_checks) > 0: - if frozenset(failed_checks) not in all_failed_checks: - all_failed_checks[frozenset(failed_checks)] = [] - all_failed_checks[frozenset(failed_checks)].append(class_name) - - if frozenset(passed_checks) not in all_passed_checks: - all_passed_checks[frozenset(passed_checks)] = [] - all_passed_checks[frozenset(passed_checks)].append(class_name) - - if len(all_failed_checks) > 0: - print("Following tests failed for components:") - for key, value in all_failed_checks.items(): - print('========================') - print(key) - print(value) - raise RuntimeError("estimator checks failed") - print("success, total checks passed %s ", total_checks_passed) - class TestEstimatorChecks(unittest.TestCase): - - def test_estimator_checks_1(self): - epoints = get_epoints() - check_entry_points(epoints[:len(epoints)//2]) - - def test_estimator_checks_2(self): - epoints = get_epoints() - check_entry_points(epoints[len(epoints)//2:]) + # This method is a static method of the class + # because there were pytest fixture related + # issues when the method was in the global scope. + @staticmethod + def generate_test_method(epoint): + def method(self): + failed_checks = set() + passed_checks = set() + class_name = epoint[1] + print("\n======== now Estimator is %s =========== " % class_name) + + mod = __import__('nimbusml.' + epoint[0], fromlist=[str(class_name)]) + the_class = getattr(mod, class_name) + if class_name in INSTANCES: + estimator = INSTANCES[class_name] + else: + estimator = the_class() + + if estimator._use_single_input_as_string(): + estimator = estimator << 'F0' + + for check in _yield_all_checks(class_name, estimator): + # Skip check_dict_unchanged for estimators which + # update the classes_ attribute. For more details + # see https://github.com/microsoft/NimbusML/pull/200 + if (check.__name__ == 'check_dict_unchanged') and \ + (hasattr(estimator, 'predict_proba') or + hasattr(estimator, 'decision_function')): + continue + + if check.__name__ in OMITTED_CHECKS_ALWAYS: + continue + if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS: + continue + if class_name in OMITTED_CHECKS and check.__name__ in \ + OMITTED_CHECKS[class_name]: + continue + if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \ + OMITTED_CHECKS_TUPLE[1]: + continue + + try: + check(class_name, estimator.clone()) + passed_checks.add(check.__name__) + except Exception as e: + failed_checks.add(check.__name__) + + if len(failed_checks) > 0: + self.fail(msg=str(failed_checks)) + + return method + + +for epoint in get_epoints(): + test_name = 'test_%s' % epoint[1].lower() + method = TestEstimatorChecks.generate_test_method(epoint) + setattr(TestEstimatorChecks, test_name, method) if __name__ == '__main__': From 9e4efffabd155241884ec93c43cdd467c6dc8d53 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 17 Oct 2019 16:36:44 -0700 Subject: [PATCH 13/40] Test intentional failure on build servers. --- src/python/nimbusml/tests/test_csr_matrix_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/test_csr_matrix_output.py b/src/python/nimbusml/tests/test_csr_matrix_output.py index f4909906..a57c8934 100644 --- a/src/python/nimbusml/tests/test_csr_matrix_output.py +++ b/src/python/nimbusml/tests/test_csr_matrix_output.py @@ -30,7 +30,7 @@ def test_column_dropped_output_produces_expected_result(self): result = pd.DataFrame(result.todense()) train_data = {0: [1, 0, 0, 4], - 1: [2, 3, 0, 5]} + 1: [2, 3, 0, 6]} expected_result = pd.DataFrame(train_data).astype(np.float32) self.assertTrue(result.equals(expected_result)) From c33fcc8dd0eb66a19bc576735e25d25d1334b849 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 21 Oct 2019 13:40:54 -0700 Subject: [PATCH 14/40] Create the test_docs_example unit tests dynamically so they can be parallelized. --- .../tests_extended/test_docs_example.py | 264 +++++++++--------- 1 file changed, 128 insertions(+), 136 deletions(-) diff --git a/src/python/tests_extended/test_docs_example.py b/src/python/tests_extended/test_docs_example.py index 1b169fe7..d60d4c34 100644 --- a/src/python/tests_extended/test_docs_example.py +++ b/src/python/tests_extended/test_docs_example.py @@ -6,78 +6,126 @@ import platform import subprocess import sys -import time import unittest import six from nimbusml import __file__ as myfile -class TestDocsExamples(unittest.TestCase): - - def test_examples(self): - this = os.path.abspath(os.path.dirname(__file__)) - fold = os.path.normpath( - os.path.join( - this, - '..', - 'nimbusml', - 'examples')) - if not os.path.exists(fold): - raise FileNotFoundError("Unable to find '{0}'.".format(fold)) - - fold_files = [(fold, _) for _ in os.listdir( - fold) if os.path.splitext(_)[-1] == '.py'] - if len(fold_files) == 0: - raise FileNotFoundError( - "Unable to find examples in '{0}'".format(fold)) - - # also include the 'examples_from_dataframe' files - fold_df = os.path.join(fold, 'examples_from_dataframe') - fold_files_df = [(fold_df, _) for _ in os.listdir( - fold_df) if os.path.splitext(_)[-1] == '.py'] - - # merge details of all examples into one list - fold_files.extend(fold_files_df) - fold_files.sort() - - modpath = os.path.abspath(os.path.dirname(myfile)) - modpath = os.path.normpath(os.path.join(os.path.join(modpath), '..')) - os.environ['PYTHONPATH'] = modpath - os.environ['PYTHONIOENCODING'] = 'UTF-8' - - ran = 0 - excs = [] - - for i, (fold, name) in enumerate(fold_files): +exps = [ + "Exception: 'Missing 'English.tok'", + "Missing resource for SSWE", + "Model file for Word Embedding transform could not " + "be found", + "was already trained. Its coefficients will be " + "overwritten. Use clone() to get an untrained " + "version of it.", + "LdaNative.dll", + "CacheClassesFromAssembly", + "Your CPU supports instructions that this TensorFlow", + "CacheClassesFromAssembly: can't map name " + "OLSLinearRegression to Void, already mapped to Void", + # TensorFlowScorer.py + "tensorflow/compiler/xla/service/service.cc:168] XLA service", + "tensorflow/compiler/xla/service/service.cc:175] StreamExecutor device", + "tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency:", + "tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU", + # Binner.py + "from collections import Mapping, defaultdict", + "DeprecationWarning: Using or importing the ABCs", + # BootStrapSample.py + "DeprecationWarning: the imp module is deprecated", + # PipelineWithGridSearchCV2.py + "FutureWarning: You should specify a value for 'cv'", + # PipelineWithGridSearchCV2.py + "DeprecationWarning: The default of the 'iid' parameter", + # PcaAnomalyDetector.py + "UserWarning: Model", + # FastLinearClassifier_iris_df.py + "FutureWarning: elementwise comparison failed", + # PcaAnomalyDetector_df.py + "FutureWarning: Sorting because non-concatenation axis", + # Image.py + "Unable to revert mtime: /Library/Fonts", + "Fontconfig error: Cannot load default config file", + ] +if sys.version_info[:2] <= (3, 6): + # This warning is new but it does not break any + # other unit tests. + # (3, 5) -> (3, 6) for tests on mac + # TODO: Investigate. + exps.append("RuntimeWarning: numpy.dtype size changed") + + +def get_examples(): + this = os.path.abspath(os.path.dirname(__file__)) + folder = os.path.normpath( + os.path.join( + this, + '..', + 'nimbusml', + 'examples')) + if not os.path.exists(folder): + raise FileNotFoundError("Unable to find '{0}'.".format(folder)) + + folder_files = [(folder, _) for _ in os.listdir( + folder) if os.path.splitext(_)[-1] == '.py'] + if len(folder_files) == 0: + raise FileNotFoundError( + "Unable to find examples in '{0}'".format(folder)) + + # also include the 'examples_from_dataframe' files + folder_df = os.path.join(folder, 'examples_from_dataframe') + folder_files_df = [(folder_df, _) for _ in os.listdir( + folder_df) if os.path.splitext(_)[-1] == '.py'] + + # merge details of all examples into one list + folder_files.extend(folder_files_df) + folder_files.sort() + + examples = [] + for folder, name in folder_files: + if name in [ + '__init__.py', + # Bug todo: CustomStopWordsRemover fails on ML.NET side + 'NGramFeaturizer2.py']: + continue + # skip for all linux tests, mac is ok + if os.name == "posix" and platform.linux_distribution()[0] != '': if name in [ - # Bug 294481: CharTokenizer_df fails - # with error about variable length vector - 'CharTokenizer_df.py', - # Bug todo: CustomStopWordsRemover fails on ML.NET side - 'NGramFeaturizer2.py', - ]: + # SymSgdNative fails to load on linux + 'SymSgdBinaryClassifier.py', + 'SymSgdBinaryClassifier_infert_df.py', + # MICROSOFTML_RESOURCE_PATH needs to be setup on linux + 'CharTokenizer.py', + 'WordEmbedding.py', + 'WordEmbedding_df.py', + 'NaiveBayesClassifier_df.py']: continue - # skip for all linux tests, mac is ok - if os.name == "posix" and platform.linux_distribution()[0] != '': - if name in [ - # SymSgdNative fails to load on linux - 'SymSgdBinaryClassifier.py', - 'SymSgdBinaryClassifier_infert_df.py', - # MICROSOFTML_RESOURCE_PATH needs to be setup on linux - 'CharTokenizer.py', - 'WordEmbedding.py', - 'WordEmbedding_df.py', - 'NaiveBayesClassifier_df.py' - ]: - continue - - full = os.path.join(fold, name) - cmd = '"{0}" -u "{1}"'.format( - sys.executable.replace( - 'w.exe', '.exe'), full) - - begin = time.clock() + + examples.append((folder, name)) + + return examples + + +class TestDocsExamples(unittest.TestCase): + # This method is a static method of the class + # because there were pytest fixture related + # issues when the method was in the global scope. + @staticmethod + def generate_test_method(folder, name): + def method(self): + print("\n======== Example: %s =========== " % name) + + modpath = os.path.abspath(os.path.dirname(myfile)) + modpath = os.path.normpath(os.path.join(os.path.join(modpath), '..')) + os.environ['PYTHONPATH'] = modpath + os.environ['PYTHONIOENCODING'] = 'UTF-8' + + full = os.path.join(folder, name) + python_exe = sys.executable.replace('w.exe', '.exe') + cmd = '"{0}" -u "{1}"'.format(python_exe, full) + if six.PY2: FNULL = open(os.devnull, 'w') p = subprocess.Popen( @@ -88,59 +136,14 @@ def test_examples(self): shell=True) stdout, stderr = p.communicate() else: - with subprocess.Popen(cmd, stdout=subprocess.PIPE, + with subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, shell=True) as p: stdout, stderr = p.communicate() - total = time.clock() - begin - stderr = stderr.decode('utf-8', errors='ignore').strip( - "\n\r\t ") - stdout = stdout.decode('utf-8', errors='ignore').strip( - "\n\r\t ") - exps = [ - "Exception: 'Missing 'English.tok'", - "Missing resource for SSWE", - "Model file for Word Embedding transform could not " - "be found", - "was already trained. Its coefficients will be " - "overwritten. Use clone() to get an untrained " - "version of it.", - "LdaNative.dll", - "CacheClassesFromAssembly", - "Your CPU supports instructions that this TensorFlow", - "CacheClassesFromAssembly: can't map name " - "OLSLinearRegression to Void, already mapped to Void", - # TensorFlowScorer.py - "tensorflow/compiler/xla/service/service.cc:168] XLA service", - "tensorflow/compiler/xla/service/service.cc:175] StreamExecutor device", - "tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency:", - "tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU", - # Binner.py - "from collections import Mapping, defaultdict", - "DeprecationWarning: Using or importing the ABCs", - # BootStrapSample.py - "DeprecationWarning: the imp module is deprecated", - # PipelineWithGridSearchCV2.py - "FutureWarning: You should specify a value for 'cv'", - # PipelineWithGridSearchCV2.py - "DeprecationWarning: The default of the 'iid' parameter", - # PcaAnomalyDetector.py - "UserWarning: Model", - # FastLinearClassifier_iris_df.py - "FutureWarning: elementwise comparison failed", - # PcaAnomalyDetector_df.py - "FutureWarning: Sorting because non-concatenation axis", - # Image.py - "Unable to revert mtime: /Library/Fonts", - "Fontconfig error: Cannot load default config file", - ] - if sys.version_info[:2] <= (3, 6): - # This warning is new but it does not break any - # other unit tests. - # (3, 5) -> (3, 6) for tests on mac - # TODO: Investigate. - exps.append("RuntimeWarning: numpy.dtype size changed") + stderr = stderr.decode('utf-8', errors='ignore').strip("\n\r\t ") + stdout = stdout.decode('utf-8', errors='ignore').strip("\n\r\t ") errors = None if stderr != '': @@ -149,25 +152,6 @@ def test_examples(self): errors = [_ for _ in errors if exp not in _] if errors and (len(errors) > 1 or (len(errors) == 1 and errors[0] != '')): - excs.append(RuntimeError( - "Issue with\n File '{0}'\n--CMD\n{1}\n--ERR\n{2}\n--OUT\n" - "{3}\n--".format(full, cmd, '\n'.join(errors), stdout))) - print("{0}/{1} FAIL - '{2}' in {3}s".format(i + 1, len( - fold_files), name, total)) - if len(excs) > 1: - for ex in excs: - print('--------------') - print(ex) - raise excs[-1] - else: - print("{0}/{1} OK - '{2}' in " - "{3}s".format(i + 1, len(fold_files), name, total)) - ran += 1 - - if len(excs) > 0: - for ex in excs[1:]: - print('--------------') - print(ex) import numpy import pandas import sklearn @@ -176,10 +160,18 @@ def test_examples(self): sklearn.__version__, numpy.__version__] print("DEBUG VERSIONS", versions) - raise excs[0] - elif ran == 0: - raise Exception( - "No example was run in path '{0}'.".format(fold)) + + raise RuntimeError( + "Issue with\n File '{0}'\n--CMD\n{1}\n--ERR\n{2}\n--OUT\n" + "{3}\n--".format(full, cmd, '\n'.join(errors), stdout)) + + return method + + +for example in get_examples(): + test_name = 'test_%s' % example[1].replace('.py', '').lower() + method = TestDocsExamples.generate_test_method(*example) + setattr(TestDocsExamples, test_name, method) if __name__ == "__main__": From 0e785bc35d1ebb62b16ad54c662f3e9e08d98c4b Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 21 Oct 2019 13:47:17 -0700 Subject: [PATCH 15/40] Update the number of concurrent extended tests. --- build.cmd | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cmd b/build.cmd index dfb40bf0..c5acdc96 100644 --- a/build.cmd +++ b/build.cmd @@ -423,7 +423,7 @@ if errorlevel 1 ( ) if "%RunExtendedTests%" == "True" ( - call "%PythonExe%" -m pytest -n 2 --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" + call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) diff --git a/build.sh b/build.sh index 9a15e379..d7567ac5 100755 --- a/build.sh +++ b/build.sh @@ -324,7 +324,7 @@ then yum install glibc-devel -y } fi - "${PythonExe}" -m pytest -n 2 --verbose --maxfail=1000 --capture=sys "${TestsPath3}" + "${PythonExe}" -m pytest -n 4 --verbose --maxfail=1000 --capture=sys "${TestsPath3}" fi fi From cd81f83f049a682d0097b340b5f9bda9e5fc85d3 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 22 Oct 2019 10:43:09 -0700 Subject: [PATCH 16/40] Remove intentional error from test_csr_matrix_output. --- src/python/nimbusml/tests/test_csr_matrix_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/test_csr_matrix_output.py b/src/python/nimbusml/tests/test_csr_matrix_output.py index a57c8934..f4909906 100644 --- a/src/python/nimbusml/tests/test_csr_matrix_output.py +++ b/src/python/nimbusml/tests/test_csr_matrix_output.py @@ -30,7 +30,7 @@ def test_column_dropped_output_produces_expected_result(self): result = pd.DataFrame(result.todense()) train_data = {0: [1, 0, 0, 4], - 1: [2, 3, 0, 6]} + 1: [2, 3, 0, 5]} expected_result = pd.DataFrame(train_data).astype(np.float32) self.assertTrue(result.equals(expected_result)) From 643b39f2e413e719590dc0ec5bc6e3334a112bc1 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 22 Oct 2019 11:24:08 -0700 Subject: [PATCH 17/40] Test intentional error in test_estimator_checks. --- src/python/tests/test_estimator_checks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index accffb7f..f320d9e5 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -326,6 +326,9 @@ def method(self): except Exception as e: failed_checks.add(check.__name__) + if class_name == "OrdinaryLeastSquaresRegressor" and check.__name__ == "check_estimators_dtypes": + failed_checks.add(check.__name__) + if len(failed_checks) > 0: self.fail(msg=str(failed_checks)) From dbec3731cec0a63bd549b7666dbf463f5de2dc54 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 22 Oct 2019 12:20:47 -0700 Subject: [PATCH 18/40] Remove the intentional error which was used for testing. --- src/python/tests/test_estimator_checks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index f320d9e5..accffb7f 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -326,9 +326,6 @@ def method(self): except Exception as e: failed_checks.add(check.__name__) - if class_name == "OrdinaryLeastSquaresRegressor" and check.__name__ == "check_estimators_dtypes": - failed_checks.add(check.__name__) - if len(failed_checks) > 0: self.fail(msg=str(failed_checks)) From fda488707f685f66c2e3258f2904fef9d0c4d7dc Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 22 Oct 2019 13:03:33 -0700 Subject: [PATCH 19/40] Add whitespace change to restart CI run. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index dc8c532f..b1d4b6b9 100644 --- a/build.cmd +++ b/build.cmd @@ -441,7 +441,7 @@ echo Failed with error %ERRORLEVEL% exit /b %ERRORLEVEL% :CleanUpDotnet -:: Save the error level so it can be +:: Save the error level so it can be :: restored when exiting the function set PrevErrorLevel=%ERRORLEVEL% From 9138a0d024d8de1cfabb9b072691e2d51a0bb96e Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Tue, 22 Oct 2019 13:49:59 -0700 Subject: [PATCH 20/40] Add whitespace change to start a new CI run. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index b1d4b6b9..dc8c532f 100644 --- a/build.cmd +++ b/build.cmd @@ -441,7 +441,7 @@ echo Failed with error %ERRORLEVEL% exit /b %ERRORLEVEL% :CleanUpDotnet -:: Save the error level so it can be +:: Save the error level so it can be :: restored when exiting the function set PrevErrorLevel=%ERRORLEVEL% From 95de2804ce35dad44b8484b9bd2a0d7b3f71d0a4 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Wed, 23 Oct 2019 11:20:14 -0700 Subject: [PATCH 21/40] Load balance by sending test grouped by file to any available environment. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index dc8c532f..fdcc4813 100644 --- a/build.cmd +++ b/build.cmd @@ -417,7 +417,7 @@ set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4 -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --dist=loadfile --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From 0495136fc2af7ba0fa8e63220d10b4bfba3a74eb Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Wed, 23 Oct 2019 11:52:17 -0700 Subject: [PATCH 22/40] Add whitespace change to start a new CI run. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index fdcc4813..3496be7a 100644 --- a/build.cmd +++ b/build.cmd @@ -441,7 +441,7 @@ echo Failed with error %ERRORLEVEL% exit /b %ERRORLEVEL% :CleanUpDotnet -:: Save the error level so it can be +:: Save the error level so it can be :: restored when exiting the function set PrevErrorLevel=%ERRORLEVEL% From 2f6aeb62aa4e99d02134a473ef848839f3d0cce6 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 09:54:31 -0700 Subject: [PATCH 23/40] Fix KMeansPlusPlus does not work with a cluster size of 1 when using a debug version of ml.net --- src/python/nimbusml/tests/model_summary/test_model_summary.py | 2 +- src/python/tests/test_estimator_checks.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/model_summary/test_model_summary.py b/src/python/nimbusml/tests/model_summary/test_model_summary.py index 650238ae..37251266 100644 --- a/src/python/nimbusml/tests/model_summary/test_model_summary.py +++ b/src/python/nimbusml/tests/model_summary/test_model_summary.py @@ -71,7 +71,7 @@ GamBinaryClassifier(), PcaAnomalyDetector(), FactorizationMachineBinaryClassifier(), - KMeansPlusPlus(), + KMeansPlusPlus(n_clusters=2), NaiveBayesClassifier(), FastForestBinaryClassifier(number_of_trees=2), FastForestRegressor(number_of_trees=2), diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index accffb7f..e04a357d 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -9,6 +9,7 @@ import os import unittest +from nimbusml.cluster import KMeansPlusPlus from nimbusml.decomposition import FactorizationMachineBinaryClassifier from nimbusml.ensemble import EnsembleClassifier from nimbusml.ensemble import EnsembleRegressor @@ -197,6 +198,7 @@ 'EnsembleClassifier': EnsembleClassifier(num_models=3), 'EnsembleRegressor': EnsembleRegressor(num_models=3), 'FactorizationMachineBinaryClassifier': FactorizationMachineBinaryClassifier(shuffle=False), + 'KMeansPlusPlus': KMeansPlusPlus(n_clusters=2), 'LightGbmBinaryClassifier': LightGbmBinaryClassifier( minimum_example_count_per_group=1, minimum_example_count_per_leaf=1), 'LightGbmClassifier': LightGbmClassifier( From 472ad3f414ab3b985400dbe3d39b1ec46195aad1 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 09:58:42 -0700 Subject: [PATCH 24/40] Fix OLS divide by 0 when given a particular set of inputs to fit. This is hidden in release versions of ml.net --- .../nimbusml/tests/model_summary/test_model_summary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/nimbusml/tests/model_summary/test_model_summary.py b/src/python/nimbusml/tests/model_summary/test_model_summary.py index 37251266..87ab897a 100644 --- a/src/python/nimbusml/tests/model_summary/test_model_summary.py +++ b/src/python/nimbusml/tests/model_summary/test_model_summary.py @@ -119,24 +119,24 @@ def test_summary_called_back_to_back_on_predictor(self): ols.summary() def test_pipeline_summary_is_refreshed_after_refitting(self): - predictor = OrdinaryLeastSquaresRegressor(normalize='No', l2_regularization=0) + predictor = OrdinaryLeastSquaresRegressor() pipeline = Pipeline([predictor]) pipeline.fit([0,1,2,3], [1,2,3,4]) summary1 = pipeline.summary() - pipeline.fit([0,1,2,3], [2,5,8,11]) + pipeline.fit([0,1,2.5,3], [2,5,8,11]) summary2 = pipeline.summary() self.assertFalse(summary1.equals(summary2)) def test_predictor_summary_is_refreshed_after_refitting(self): - predictor = OrdinaryLeastSquaresRegressor(normalize='No', l2_regularization=0) + predictor = OrdinaryLeastSquaresRegressor() predictor.fit([0,1,2,3], [1,2,3,4]) summary1 = predictor.summary() - predictor.fit([0,1,2,3], [2,5,8,11]) + predictor.fit([0,1,2.5,3], [2,5,8,11]) summary2 = predictor.summary() self.assertFalse(summary1.equals(summary2)) From b1ccc3a0ddc304db50da23ef04db88fe6339fa5c Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 10:30:09 -0700 Subject: [PATCH 25/40] Fix issue when ranking where the output of TextToKeyConverter was trying to overwrite the $scoredVectorData variable set by DatasetScorerEx. See test_metrics_evaluate_ranking_group_id_from_existing_column_in_X for a test which demonstrates the issue. It throws an exception from EntryPointNode.cs:837 when trying to get the outputs. The exception was hidden when using release builds of ML.Net. --- src/python/nimbusml/pipeline.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 704622a4..71ee437d 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1529,10 +1529,14 @@ def _evaluation_infer(self, evaltype, label_column, group_id, models_anomalydetectionevaluator(**params)]) elif type_ == 'ranking': - svd = "$scoredVectorData" column = [OrderedDict(Source=group_id, Name=group_id)] - algo_args = dict(data=svd, output_data=svd, column=column) + algo_args = dict( + data="$scoredVectorData", + output_data="$scoredVectorData2", + column=column) key_node = transforms_texttokeyconverter(**algo_args) + + params['data'] = "$scoredVectorData2" evaluate_node = models_rankingevaluator( group_id_column=group_id, **params) all_nodes.extend([ From be8835fdd122b06d9f99b39137fe8a2ef167b42e Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 11:24:18 -0700 Subject: [PATCH 26/40] Remove a test_estimator_check for OrdinaryLeastSquaresRegressor since it is causing invalid float values and throwing an exception which was hidden in release versions of ML.Net but visible in debug. --- src/python/tests/test_estimator_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index e04a357d..d8a19e1f 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -157,6 +157,7 @@ 'check_estimators_overwrite_params, \ check_estimator_sparse_data, check_estimators_pickle, ' 'check_estimators_nan_inf', + 'OrdinaryLeastSquaresRegressor': 'check_fit2d_1sample' } OMITTED_CHECKS_TUPLE = ( From 3de74fedbcf1bbd613330c102bf906a0e8360f28 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 12:54:44 -0700 Subject: [PATCH 27/40] Update test_permutation_feature_importance tests to support parallel execution. --- .../test_permutation_feature_importance.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/python/nimbusml/tests/pipeline/test_permutation_feature_importance.py b/src/python/nimbusml/tests/pipeline/test_permutation_feature_importance.py index 347b2798..04f1bc35 100644 --- a/src/python/nimbusml/tests/pipeline/test_permutation_feature_importance.py +++ b/src/python/nimbusml/tests/pipeline/test_permutation_feature_importance.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. # -------------------------------------------------------------------------------------------- import os +import tempfile import unittest from nimbusml import FileDataStream @@ -16,6 +17,14 @@ from numpy.testing import assert_almost_equal from pandas.testing import assert_frame_equal + +def get_temp_model_file(): + fd, file_name = tempfile.mkstemp(suffix='.zip') + fl = os.fdopen(fd, 'w') + fl.close() + return file_name + + class TestPermutationFeatureImportance(unittest.TestCase): @classmethod @@ -65,7 +74,7 @@ def test_binary_classifier(self): assert_almost_equal(self.binary_pfi['AreaUnderPrecisionRecallCurve'].sum(), -0.19365, 5) def test_binary_classifier_from_loaded_model(self): - model_path = "model.zip" + model_path = get_temp_model_file() self.binary_model.save_model(model_path) loaded_model = Pipeline() loaded_model.load_model(model_path) @@ -81,7 +90,7 @@ def test_clasifier(self): assert_almost_equal(self.classifier_pfi['PerClassLogLoss.1'].sum(), 0.419826, 6) def test_classifier_from_loaded_model(self): - model_path = "model.zip" + model_path = get_temp_model_file() self.classifier_model.save_model(model_path) loaded_model = Pipeline() loaded_model.load_model(model_path) @@ -96,7 +105,7 @@ def test_regressor(self): assert_almost_equal(self.regressor_pfi['RSquared'].sum(), -0.203612, 6) def test_regressor_from_loaded_model(self): - model_path = "model.zip" + model_path = get_temp_model_file() self.regressor_model.save_model(model_path) loaded_model = Pipeline() loaded_model.load_model(model_path) @@ -113,7 +122,7 @@ def test_ranker(self): assert_almost_equal(self.ranker_pfi['NDCG@3'].sum(), -0.236544, 6) def test_ranker_from_loaded_model(self): - model_path = "model.zip" + model_path = get_temp_model_file() self.ranker_model.save_model(model_path) loaded_model = Pipeline() loaded_model.load_model(model_path) From 75df29384cd03bd192e2f4cae57873e1609e8c22 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 13:23:02 -0700 Subject: [PATCH 28/40] Remove --dist=loadfile from the windows unit test run. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index 3496be7a..b1d4b6b9 100644 --- a/build.cmd +++ b/build.cmd @@ -417,7 +417,7 @@ set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4 -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --dist=loadfile --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From 7122e1972e48c182a10649b8d63319f7c24829b2 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 14:34:00 -0700 Subject: [PATCH 29/40] Update test_load_save to support parallel execution. --- .../nimbusml/tests/pipeline/test_load_save.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 19bc26ce..3807507e 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -5,6 +5,7 @@ import os import pickle +import tempfile import unittest import numpy as np @@ -32,6 +33,12 @@ (train, label) = get_X_y(train_file, label_column, sep=',') (test, test_label) = get_X_y(test_file, label_column, sep=',') +def get_temp_file(suffix=None): + fd, file_name = tempfile.mkstemp(suffix=suffix) + fl = os.fdopen(fd, 'w') + fl.close() + return file_name + class TestLoadSave(unittest.TestCase): @@ -48,7 +55,7 @@ def test_model_dataframe(self): model_nimbusml.fit(train, label) # Save with pickle - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') with open(pickle_filename, 'wb') as f: pickle.dump(model_nimbusml, f) @@ -65,9 +72,10 @@ def test_model_dataframe(self): test, test_label, output_scores=True) # Save load with pipeline methods - model_nimbusml.save_model('model.nimbusml.m') + model_filename = get_temp_file(suffix='.m') + model_nimbusml.save_model(model_filename) model_nimbusml_load = Pipeline() - model_nimbusml_load.load_model('model.nimbusml.m') + model_nimbusml_load.load_model(model_filename) score1 = model_nimbusml.predict(test).head(5) score2 = model_nimbusml_load.predict(test).head(5) @@ -82,7 +90,7 @@ def test_model_dataframe(self): model_nimbusml_load.sum().sum(), decimal=2) - os.remove('model.nimbusml.m') + os.remove(model_filename) def test_model_datastream(self): model_nimbusml = Pipeline( @@ -97,7 +105,7 @@ def test_model_datastream(self): model_nimbusml.fit(train, label) # Save with pickle - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') with open(pickle_filename, 'wb') as f: pickle.dump(model_nimbusml, f) @@ -120,9 +128,10 @@ def test_model_datastream(self): decimal=2) # Save load with pipeline methods - model_nimbusml.save_model('model.nimbusml.m') + model_filename = get_temp_file(suffix='.m') + model_nimbusml.save_model(model_filename) model_nimbusml_load = Pipeline() - model_nimbusml_load.load_model('model.nimbusml.m') + model_nimbusml_load.load_model(model_filename) score1 = model_nimbusml.predict(test).head(5) score2 = model_nimbusml_load.predict(test).head(5) @@ -137,7 +146,7 @@ def test_model_datastream(self): model_nimbusml_load.sum().sum(), decimal=2) - os.remove('model.nimbusml.m') + os.remove(model_filename) def test_pipeline_saves_complete_model_file_when_pickled(self): model_nimbusml = Pipeline( @@ -152,7 +161,7 @@ def test_pipeline_saves_complete_model_file_when_pickled(self): model_nimbusml.fit(train, label) metrics, score = model_nimbusml.test(test, test_label, output_scores=True) - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') # Save with pickle with open(pickle_filename, 'wb') as f: @@ -202,7 +211,7 @@ def test_unfitted_pickled_pipeline_can_be_fit(self): shuffle=False, number_of_threads=1))]) - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') # Save with pickle with open(pickle_filename, 'wb') as f: @@ -234,7 +243,7 @@ def test_unpickled_pipeline_has_feature_contributions(self): fc = model_nimbusml.get_feature_contributions(test) # Save with pickle - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') with open(pickle_filename, 'wb') as f: pickle.dump(model_nimbusml, f) # Unpickle model @@ -260,7 +269,7 @@ def test_unpickled_predictor_has_feature_contributions(self): fc = model_nimbusml.get_feature_contributions(test) # Save with pickle - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') with open(pickle_filename, 'wb') as f: pickle.dump(model_nimbusml, f) # Unpickle model @@ -287,7 +296,7 @@ def test_pipeline_loaded_from_zip_has_feature_contributions(self): fc = model_nimbusml.get_feature_contributions(test) # Save the model to zip - model_filename = 'nimbusml_model.zip' + model_filename = get_temp_file(suffix='.zip') model_nimbusml.save_model(model_filename) # Load the model from zip model_nimbusml_zip = Pipeline() @@ -312,7 +321,7 @@ def test_predictor_loaded_from_zip_has_feature_contributions(self): fc = model_nimbusml.get_feature_contributions(test) # Save the model to zip - model_filename = 'nimbusml_model.zip' + model_filename = get_temp_file(suffix='.zip') model_nimbusml.save_model(model_filename) # Load the model from zip model_nimbusml_zip = Pipeline() @@ -347,7 +356,7 @@ def test_pickled_pipeline_with_predictor_model(self): self.assertTrue(pipeline.predictor_model) self.assertNotEqual(pipeline.model, pipeline.predictor_model) - pickle_filename = 'nimbusml_model.p' + pickle_filename = get_temp_file(suffix='.p') with open(pickle_filename, 'wb') as f: pickle.dump(pipeline, f) From dbf6c47d933af484d3dddeb00adb5e47f3ecf2bd Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 14:51:17 -0700 Subject: [PATCH 30/40] Test turning off pytest assert rewriting. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index b1d4b6b9..d6118493 100644 --- a/build.cmd +++ b/build.cmd @@ -417,7 +417,7 @@ set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4 -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) From 366b1b0d39b955d48f507a1b9aa33d2d6f775165 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 15:30:26 -0700 Subject: [PATCH 31/40] Test turning off forcing at least 4 concurrent unit tests --- build.cmd | 1 - 1 file changed, 1 deletion(-) diff --git a/build.cmd b/build.cmd index d6118493..a225a841 100644 --- a/build.cmd +++ b/build.cmd @@ -415,7 +415,6 @@ set TestsPath2=%__currentScriptDir%src\python\tests set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% -if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4 call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( From c345a0f7aedd73ebb02ddc5a447c8baee106d684 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 15:57:19 -0700 Subject: [PATCH 32/40] Whitespace change to start a new CI run. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index a225a841..fbe4e05c 100644 --- a/build.cmd +++ b/build.cmd @@ -408,7 +408,7 @@ if "%RunTests%" == "False" ( echo "" echo "#################################" echo "Running tests ... " -echo "#################################" +echo "#################################" set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests From e6804ea7d65519938238049a40fb6c32778422bc Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Thu, 24 Oct 2019 16:25:29 -0700 Subject: [PATCH 33/40] Test tests run without coverage. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index fbe4e05c..7d0fab55 100644 --- a/build.cmd +++ b/build.cmd @@ -416,7 +416,7 @@ set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" if errorlevel 1 ( goto :Exit_Error ) From 4cf17cddc566431a7dd76cf5ff57f84f056f4a4e Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Fri, 25 Oct 2019 10:04:47 -0700 Subject: [PATCH 34/40] Try and capture the test order during the CI run. --- build.cmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cmd b/build.cmd index 7d0fab55..3cae8902 100644 --- a/build.cmd +++ b/build.cmd @@ -388,7 +388,7 @@ if "%InstallPythonPackages%" == "True" ( echo "Installing python packages ... " echo "#################################" call "%PythonExe%" -m pip install --upgrade pip - call "%PythonExe%" -m pip install --upgrade nose pytest pytest-xdist graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" + call "%PythonExe%" -m pip install --upgrade nose pytest pytest-xdist pytest-replay graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq @@ -416,7 +416,7 @@ set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --replay-record-dir=target --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" if errorlevel 1 ( goto :Exit_Error ) From 3988ac4d2270fb29ab39b8f63e729cb6fbc2ba11 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Fri, 25 Oct 2019 13:54:07 -0700 Subject: [PATCH 35/40] Remove the pydist replay request. --- build.cmd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.cmd b/build.cmd index 3cae8902..fcd11987 100644 --- a/build.cmd +++ b/build.cmd @@ -388,7 +388,7 @@ if "%InstallPythonPackages%" == "True" ( echo "Installing python packages ... " echo "#################################" call "%PythonExe%" -m pip install --upgrade pip - call "%PythonExe%" -m pip install --upgrade nose pytest pytest-xdist pytest-replay graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" + call "%PythonExe%" -m pip install --upgrade nose pytest pytest-xdist graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0" if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq @@ -408,7 +408,7 @@ if "%RunTests%" == "False" ( echo "" echo "#################################" echo "Running tests ... " -echo "#################################" +echo "#################################" set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml set TestsPath1=%PackagePath%\tests set TestsPath2=%__currentScriptDir%src\python\tests @@ -416,7 +416,7 @@ set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --replay-record-dir=target --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" if errorlevel 1 ( goto :Exit_Error ) From ed10974c2af17a4fd2245bf2326aba64e624d3ba Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Fri, 25 Oct 2019 16:31:38 -0700 Subject: [PATCH 36/40] Rerun unit tests one extra time if any failed to check for intermittent failures. --- build.cmd | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index fcd11987..1c9586dd 100644 --- a/build.cmd +++ b/build.cmd @@ -418,7 +418,12 @@ set NumConcurrentTests=%NUMBER_OF_PROCESSORS% call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" if errorlevel 1 ( - goto :Exit_Error + :: Rerun any failed tests to give them one more + :: chance in case the errors were intermittent. + call "%PythonExe%" -m pytest -n %NumConcurrentTests% --last-failed --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" + if errorlevel 1 ( + goto :Exit_Error + ) ) if "%RunExtendedTests%" == "True" ( From 8161d2475093128306a916e746bd74f86b02c6fb Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 28 Oct 2019 09:35:36 -0700 Subject: [PATCH 37/40] Turn back on assert rewriting and coverage reporting. Run extended tests a second time to check for intermittent failures. --- build.cmd | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build.cmd b/build.cmd index 1c9586dd..a9013e5d 100644 --- a/build.cmd +++ b/build.cmd @@ -416,11 +416,11 @@ set TestsPath3=%__currentScriptDir%src\python\tests_extended set ReportPath=%__currentScriptDir%build\TestCoverageReport set NumConcurrentTests=%NUMBER_OF_PROCESSORS% -call "%PythonExe%" -m pytest -n %NumConcurrentTests% --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" +call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( :: Rerun any failed tests to give them one more :: chance in case the errors were intermittent. - call "%PythonExe%" -m pytest -n %NumConcurrentTests% --last-failed --assert=plain --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" + call "%PythonExe%" -m pytest -n %NumConcurrentTests% --last-failed --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( goto :Exit_Error ) @@ -429,7 +429,12 @@ if errorlevel 1 ( if "%RunExtendedTests%" == "True" ( call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" if errorlevel 1 ( - goto :Exit_Error + :: Rerun any failed tests to give them one more + :: chance in case the errors were intermittent. + call "%PythonExe%" -m pytest -n %NumConcurrentTests% --last-failed --verbose --maxfail=1000 --capture=sys "%TestsPath3%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%" + if errorlevel 1 ( + goto :Exit_Error + ) ) ) From c3c4dbd0b306e64eba29339ba3c1f8deefbc2ecb Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 28 Oct 2019 10:14:52 -0700 Subject: [PATCH 38/40] Remove whitespace at the end of the comment in build.cmd. --- build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.cmd b/build.cmd index a9013e5d..55d6e937 100644 --- a/build.cmd +++ b/build.cmd @@ -450,7 +450,7 @@ echo Failed with error %ERRORLEVEL% exit /b %ERRORLEVEL% :CleanUpDotnet -:: Save the error level so it can be +:: Save the error level so it can be :: restored when exiting the function set PrevErrorLevel=%ERRORLEVEL% From c7bb35386bf6b3787de0222446dede87a1e639dc Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 28 Oct 2019 10:24:44 -0700 Subject: [PATCH 39/40] Test errors in tests show up correctly in CI build output. --- src/python/nimbusml/tests/model_summary/test_model_summary.py | 1 + src/python/nimbusml/tests/test_csr_matrix_output.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/model_summary/test_model_summary.py b/src/python/nimbusml/tests/model_summary/test_model_summary.py index 87ab897a..f1332d3e 100644 --- a/src/python/nimbusml/tests/model_summary/test_model_summary.py +++ b/src/python/nimbusml/tests/model_summary/test_model_summary.py @@ -129,6 +129,7 @@ def test_pipeline_summary_is_refreshed_after_refitting(self): summary2 = pipeline.summary() self.assertFalse(summary1.equals(summary2)) + self.assertTrue(False) def test_predictor_summary_is_refreshed_after_refitting(self): predictor = OrdinaryLeastSquaresRegressor() diff --git a/src/python/nimbusml/tests/test_csr_matrix_output.py b/src/python/nimbusml/tests/test_csr_matrix_output.py index f4909906..72e128dd 100644 --- a/src/python/nimbusml/tests/test_csr_matrix_output.py +++ b/src/python/nimbusml/tests/test_csr_matrix_output.py @@ -49,7 +49,8 @@ def test_fit_transform_produces_expected_result(self): result = pd.DataFrame(result.todense()) train_data = {0: [1, 0, 0, 4], - 1: [2, 3, 0, 5]} + #1: [2, 3, 0, 5]} + 1: [2, 3, 1, 5]} expected_result = pd.DataFrame(train_data).astype(np.float32) self.assertTrue(result.equals(expected_result)) From 8aeab82a243e57abc7bcffb6fef8757ddec5d3cc Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Mon, 28 Oct 2019 10:50:43 -0700 Subject: [PATCH 40/40] Revert the intentional test failures from the previous commit. --- src/python/nimbusml/tests/model_summary/test_model_summary.py | 1 - src/python/nimbusml/tests/test_csr_matrix_output.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/python/nimbusml/tests/model_summary/test_model_summary.py b/src/python/nimbusml/tests/model_summary/test_model_summary.py index f1332d3e..87ab897a 100644 --- a/src/python/nimbusml/tests/model_summary/test_model_summary.py +++ b/src/python/nimbusml/tests/model_summary/test_model_summary.py @@ -129,7 +129,6 @@ def test_pipeline_summary_is_refreshed_after_refitting(self): summary2 = pipeline.summary() self.assertFalse(summary1.equals(summary2)) - self.assertTrue(False) def test_predictor_summary_is_refreshed_after_refitting(self): predictor = OrdinaryLeastSquaresRegressor() diff --git a/src/python/nimbusml/tests/test_csr_matrix_output.py b/src/python/nimbusml/tests/test_csr_matrix_output.py index 72e128dd..f4909906 100644 --- a/src/python/nimbusml/tests/test_csr_matrix_output.py +++ b/src/python/nimbusml/tests/test_csr_matrix_output.py @@ -49,8 +49,7 @@ def test_fit_transform_produces_expected_result(self): result = pd.DataFrame(result.todense()) train_data = {0: [1, 0, 0, 4], - #1: [2, 3, 0, 5]} - 1: [2, 3, 1, 5]} + 1: [2, 3, 0, 5]} expected_result = pd.DataFrame(train_data).astype(np.float32) self.assertTrue(result.equals(expected_result))