From 230c2cfc98ef3b0a0e2c97b831fa9832c6805b58 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Wed, 11 Mar 2020 12:38:27 -0700 Subject: [PATCH 1/2] Upgrade CI build to use latest onnxruntime and automl scenario based test --- build.cmd | 1 + build.sh | 1 + src/python/nimbusml.pyproj | 2 +- src/python/setup.py | 2 + src/python/setup.py.in | 2 + .../tests_extended/test_automl_scenario.py | 104 ++++++++++++++++++ src/python/tests_extended/vinod.py | 80 -------------- 7 files changed, 111 insertions(+), 81 deletions(-) create mode 100644 src/python/tests_extended/test_automl_scenario.py delete mode 100644 src/python/tests_extended/vinod.py diff --git a/build.cmd b/build.cmd index 46c56ac5..233a4eed 100644 --- a/build.cmd +++ b/build.cmd @@ -400,6 +400,7 @@ if "%InstallPythonPackages%" == "True" ( call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%" call "%PythonExe%" -m pip install "scikit-learn==0.19.2" + call "%PythonExe%" -m pip install --upgrade onnxruntime ) if "%RunTests%" == "False" ( diff --git a/build.sh b/build.sh index 2b20be39..1e523afa 100755 --- a/build.sh +++ b/build.sh @@ -296,6 +296,7 @@ then fi "${PythonExe}" -m pip install --upgrade "${Wheel}" "${PythonExe}" -m pip install "scikit-learn==0.19.2" + "${PythonExe}" -m pip install --upgrade onnxruntime fi if [ ${__runTests} = true ] diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 971fe420..2deae4ab 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -755,7 +755,7 @@ - + diff --git a/src/python/setup.py b/src/python/setup.py index b1279104..9ba7ff88 100644 --- a/src/python/setup.py +++ b/src/python/setup.py @@ -114,6 +114,7 @@ 'tests': [ 'nose>=1.3', 'pytest>=4.4.0', 'graphviz', 'imageio', + 'onnxruntime', ], 'dprep': ['azureml-dataprep>=1.1.33'], 'utils': ['graphviz', 'imageio'], @@ -134,6 +135,7 @@ 'nbconvert>=4.2.0', 'nose>=1.3', 'pytest>=4.4.0', + 'onnxruntime', ], python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <3.8.*', diff --git a/src/python/setup.py.in b/src/python/setup.py.in index 0489bc13..a995460d 100644 --- a/src/python/setup.py.in +++ b/src/python/setup.py.in @@ -114,6 +114,7 @@ setup( 'tests': [ 'nose>=1.3', 'pytest>=4.4.0', 'graphviz', 'imageio', + 'onnxruntime', ], 'dprep': ['azureml-dataprep>=1.1.33'], 'utils': ['graphviz', 'imageio'], @@ -134,6 +135,7 @@ setup( 'nbconvert>=4.2.0', 'nose>=1.3', 'pytest>=4.4.0', + 'onnxruntime' ], python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <3.8.*', diff --git a/src/python/tests_extended/test_automl_scenario.py b/src/python/tests_extended/test_automl_scenario.py new file mode 100644 index 00000000..02010357 --- /dev/null +++ b/src/python/tests_extended/test_automl_scenario.py @@ -0,0 +1,104 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- + +import os +import time +import tempfile +import unittest +import pandas as pd +from nimbusml.datasets import get_dataset +from nimbusml.linear_model import AveragedPerceptronBinaryClassifier +from nimbusml.feature_extraction.categorical import OneHotVectorizer +from nimbusml.feature_extraction.text import NGramFeaturizer +from nimbusml.preprocessing.missing_values import Handler +from nimbusml import FileDataStream +from nimbusml.preprocessing import DatasetTransformer +from nimbusml.preprocessing.schema import ColumnSelector +from nimbusml import Pipeline +from nimbusml.preprocessing import OnnxRunner +from data_frame_tool import DataFrameTool as DFT +from nimbusml.multiclass import OneVsRestClassifier + + +def get_tmp_file(suffix=None): + fd, file_name = tempfile.mkstemp(suffix=suffix) + fl = os.fdopen(fd, 'w') + fl.close() + return file_name + +path = get_dataset("wiki_detox_train").as_filepath() +train_set = FileDataStream.read_csv(path, sep='\t') +path = get_dataset("wiki_detox_test").as_filepath() +test_set = FileDataStream.read_csv(path, sep='\t') + +class TestOnnxRuntime(unittest.TestCase): + """ + Tests automl use case: + 1. Fit featurization pipeline separately. + 2. Fit learner on top of the featurization pipeline. + 3. Export to ONNX the learner pipeline. + 4. Compare results between ML.NET and ORT + """ + + def test_automl_usecase(self): + # train featurization pipeline + featurization_pipe = Pipeline([NGramFeaturizer(keep_diacritics=True, columns={'Features': ['SentimentText']})]) + featurization_pipe.fit(train_set) + + # train learner pipeline + learner_pipe = Pipeline([DatasetTransformer(featurization_pipe.model), + OneVsRestClassifier(AveragedPerceptronBinaryClassifier(), + feature=['Features'], label='Sentiment') + ]) + learner_pipe.fit(train_set) + + # Export the learner pipeline to ONNX + onnx_path = get_tmp_file('.onnx') + learner_pipe.export_to_onnx(onnx_path, 'com.microsoft.ml', onnx_version='Stable') + + # Perform the transform using the standard ML.Net backend + start = time.time() + result_standard = learner_pipe.predict(test_set) + end = time.time() + print('%ss done transform using standard backend' % round(end - start, 3)) + + # Perform the transform using the ORT backend + df_tool = DFT(onnx_path) + dataset = test_set.to_df() + start = time.time() + result_ort = df_tool.execute(dataset, ['PredictedLabel.output', 'Score.output']) + end = time.time() + print('%ss done transform using ORT backend (excludes df load time)' % round(end - start, 3)) + + # compare the results + for col_tuple in (('PredictedLabel', 'PredictedLabel.output'), + ('Score.0', 'Score.output.0'), + ('Score.1', 'Score.output.1'), + ): + try: + col_expected = result_standard.loc[:, col_tuple[0]] + col_ort = result_ort.loc[:, col_tuple[1]] + + if isinstance(col_expected.dtype, pd.api.types.CategoricalDtype): + # ONNX does not export categorical columns so convert categorical + # columns received from ML.Net back to the original values before + # the comparison. + col_expected = col_expected.astype(col_expected.dtype.categories.dtype) + + check_kwargs = { + 'check_names': False, + 'check_exact': False, + 'check_dtype': True, + 'check_less_precise': True + } + + pd.testing.assert_series_equal(col_expected, col_ort, **check_kwargs) + + except Exception as e: + print(e) + raise RuntimeError("ERROR: OnnxRunner result does not match expected result.") + +if __name__ == '__main__': + unittest.main() diff --git a/src/python/tests_extended/vinod.py b/src/python/tests_extended/vinod.py deleted file mode 100644 index 28a62a16..00000000 --- a/src/python/tests_extended/vinod.py +++ /dev/null @@ -1,80 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -------------------------------------------------------------------------------------------- - -import os -import time -import tempfile -import nimbusml.linear_model as nml_linear -from nimbusml.feature_extraction.categorical import OneHotVectorizer -from nimbusml.preprocessing.missing_values import Handler -from nimbusml import FileDataStream -from nimbusml.preprocessing import DatasetTransformer -from nimbusml.preprocessing.schema import ColumnSelector -from nimbusml import Pipeline -from nimbusml.preprocessing import OnnxRunner -from data_frame_tool import DataFrameTool as DFT - -def get_tmp_file(suffix=None): - fd, file_name = tempfile.mkstemp(suffix=suffix) - fl = os.fdopen(fd, 'w') - fl.close() - return file_name - -X_train_dprep = FileDataStream.read_csv("E:/sources/vinod/NYCTaxiTipPrediction_train.csv") -X_test_dprep = FileDataStream.read_csv("E:/sources/vinod/NYCTaxiTipPrediction_valid.csv") - -try: - pipe_featurization = Pipeline([OneHotVectorizer(columns={'vendor_id': 'vendor_id', 'payment_type': 'payment_type', 'passenger_count': 'passenger_count','rate_code': 'rate_code'}) - ,Handler(columns={'trip_distance': 'trip_distance', 'trip_time_in_secs': 'trip_time_in_secs'}) - ]) - pipe_featurization.fit(X_train_dprep) - - pipe_training = Pipeline([DatasetTransformer(pipe_featurization.model), - nml_linear.FastLinearRegressor(feature=['vendor_id', 'payment_type', 'passenger_count', 'rate_code', 'trip_distance', 'trip_time_in_secs'],label='fare_amount') - ]) - pipe_training.fit(X_train_dprep) - - metrics, scores = pipe_training.test(X_test_dprep) - print(metrics) - print('training done') - - # Export the pipeline to ONNX - onnx_path = get_tmp_file('.onnx') - pipe_training.export_to_onnx(onnx_path, 'com.microsoft.ml', onnx_version='Stable') - print('export done') - - # Perform the transform using the standard ML.Net backend - start = time.time() - result_standard = pipe_training.predict(X_test_dprep) - end = time.time() - print(result_standard) - print('%ss done transform using standard backend' % round(end - start, 3)) - - # Perform the transform using the ONNX backend. - # Note, the extra columns and column name differences - # is a known issue with the ML.Net backend. - onnxrunner = Pipeline([OnnxRunner(model_file=onnx_path), - ColumnSelector(columns=['Score'])]) - # Performance issue, commenting out for now - #start = time.time() - #result_onnx = onnxrunner.fit_transform(X_test_dprep, as_binary_data_stream=True) - #end = time.time() - #print(result_onnx.head(5)) - #print('%ss done transform using onnx backend' % round(end - start, 3)) - - df_tool = DFT(onnx_path) - dataset = X_test_dprep.to_df() - start = time.time() - result_ort = df_tool.execute(dataset, []) - end = time.time() - print(result_ort) - print('%ss done transform using ORT backend (excludes df load time)' % round(end - start, 3)) - - -except Exception as e: - print('=============== ERROR =================') - print(e) - -print ("done") \ No newline at end of file From 22531eaee4d79b40454b9bd1095ba2707014c2f4 Mon Sep 17 00:00:00 2001 From: Gani Nazirov Date: Wed, 11 Mar 2020 12:44:36 -0700 Subject: [PATCH 2/2] simplify --- .../tests_extended/test_automl_scenario.py | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/python/tests_extended/test_automl_scenario.py b/src/python/tests_extended/test_automl_scenario.py index 02010357..ec40ff37 100644 --- a/src/python/tests_extended/test_automl_scenario.py +++ b/src/python/tests_extended/test_automl_scenario.py @@ -8,18 +8,13 @@ import tempfile import unittest import pandas as pd +from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import AveragedPerceptronBinaryClassifier -from nimbusml.feature_extraction.categorical import OneHotVectorizer from nimbusml.feature_extraction.text import NGramFeaturizer -from nimbusml.preprocessing.missing_values import Handler -from nimbusml import FileDataStream +from nimbusml.linear_model import AveragedPerceptronBinaryClassifier +from nimbusml.multiclass import OneVsRestClassifier from nimbusml.preprocessing import DatasetTransformer -from nimbusml.preprocessing.schema import ColumnSelector -from nimbusml import Pipeline -from nimbusml.preprocessing import OnnxRunner from data_frame_tool import DataFrameTool as DFT -from nimbusml.multiclass import OneVsRestClassifier def get_tmp_file(suffix=None): @@ -77,28 +72,17 @@ def test_automl_usecase(self): ('Score.0', 'Score.output.0'), ('Score.1', 'Score.output.1'), ): - try: - col_expected = result_standard.loc[:, col_tuple[0]] - col_ort = result_ort.loc[:, col_tuple[1]] - - if isinstance(col_expected.dtype, pd.api.types.CategoricalDtype): - # ONNX does not export categorical columns so convert categorical - # columns received from ML.Net back to the original values before - # the comparison. - col_expected = col_expected.astype(col_expected.dtype.categories.dtype) - - check_kwargs = { - 'check_names': False, - 'check_exact': False, - 'check_dtype': True, - 'check_less_precise': True - } + col_expected = result_standard.loc[:, col_tuple[0]] + col_ort = result_ort.loc[:, col_tuple[1]] - pd.testing.assert_series_equal(col_expected, col_ort, **check_kwargs) + check_kwargs = { + 'check_names': False, + 'check_exact': False, + 'check_dtype': True, + 'check_less_precise': True + } - except Exception as e: - print(e) - raise RuntimeError("ERROR: OnnxRunner result does not match expected result.") + pd.testing.assert_series_equal(col_expected, col_ort, **check_kwargs) if __name__ == '__main__': unittest.main()