diff --git a/backends/openvino/tests/README.md b/backends/openvino/tests/README.md new file mode 100644 index 00000000000..3ad109274f5 --- /dev/null +++ b/backends/openvino/tests/README.md @@ -0,0 +1,67 @@ +# Unit Tests for OpenVINO Backend + +## Directory Structure + +Below is the layout of the `backends/openvino/tests` directory, which includes the necessary files for the example applications: + +``` +backends/openvino/tests +├── ops # Directory with base op test script and individual op tests. + ├── base_openvino_op_test.py # Script which contains the base class for all op tests. + └── test_.py # Individual op tests scripts. +├── models # Directory with model test scripts. + └── test_classification.py # Test script for classification models. +├── README.md # Documentation for unit tests (this file) +└── test_runner.py # Script to execute unit tests. +``` + +## Executing Unit Tests + +### Prerequisites + +Before you begin, refer to instructions provided in [OpenVINO Backend for ExecuTorch](../README.md) to install openvino and setup executorch environment. +Once openvino is installed and executorch environment is set, refer to [OpenVINO Backend Examples](../../../examples/openvino/README.md) to build openvino_example_runner. + +### Usage + +`test_runner.py` allows to run op or model tests for openvino backend. + +### **Arguments** +- **`--build_folder`** (required): + Path to cmake binary directory. (Refer to [OpenVINO Backend Examples](../../../examples/openvino/README.md)) + Examples: + - `../../../cmake-openvino-out` (Relative path from `backends/openvino/tests` directory) + - `/cmake-openvino-out` (Absolute path to the default build folder) + +- **`--test_type`** (optional): + Type of the tests to run. + Supported values: + - `ops` (default) + - `models` + +- **`--pattern`** (optional): + Pattern to match test files. Provide complete file name to run individual tests. The default value is `test_*.py` + Examples: + - `test_convolution.py` (Assuming `--test_type` parameter is provided as `ops`, this will run only convolution tests) + - `test_add*.py` (Assuming `--test_type` parameter is provided as `ops`, this will run add and addmm op tests) + +- **`--device`** (optional): + Target device to compile and run tests. Default is `CPU`. + Examples: `CPU`, `GPU` + + +## **Examples** + +### Execute Tests for All Ops on CPU +```bash +python test_runner.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops +``` + +### Execute Convolution Op Tests on CPU +```bash +python test_runner.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops --pattern test_convolution.py +``` + +### Execute Tests for all Models on GPU +```bash +python test_runner.py --build_folder ../../../cmake-openvino-out --device GPU --test_type models diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index a51b99e8eca..46c0b63fc37 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -97,13 +97,10 @@ def execute_layer_test( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, - cwd=tmp_dir, ) - stdout_str = proc.stdout.decode('utf-8') - # Check if execution completed successfully - self.assertIn("Model executed successfully.", stdout_str) + self.assertTrue(proc.returncode == 0) # Read the outputs from the temporary files output_dir = f"{tmp_dir}/outputs" diff --git a/backends/openvino/tests/test_openvino_delegate.py b/backends/openvino/tests/test_runner.py similarity index 91% rename from backends/openvino/tests/test_openvino_delegate.py rename to backends/openvino/tests/test_runner.py index eaabcf2603b..89763d1d960 100644 --- a/backends/openvino/tests/test_openvino_delegate.py +++ b/backends/openvino/tests/test_runner.py @@ -71,4 +71,8 @@ def parse_arguments(): # Discover all existing op tests in "ops" folder suite = loader.discover(test_params['test_type'], pattern=test_params['pattern']) # Start running tests - unittest.TextTestRunner().run(suite) + result = unittest.TextTestRunner().run(suite) + if result.wasSuccessful(): + print("OpenVINO backend tests completed successfully") + else: + print("OpenVINO backend tests completed with failures") diff --git a/examples/openvino/aot/aot_openvino_compiler.py b/examples/openvino/aot/aot_openvino_compiler.py index f0844289580..b7bf63a386a 100644 --- a/examples/openvino/aot/aot_openvino_compiler.py +++ b/examples/openvino/aot/aot_openvino_compiler.py @@ -20,7 +20,7 @@ from executorch.backends.openvino import OpenVINOQuantizer from executorch.backends.openvino.partitioner import OpenvinoPartitioner from executorch.exir import EdgeProgramManager -from executorch.exir import to_edge +from executorch.exir import to_edge_transform_and_lower from executorch.exir.backend.backend_details import CompileSpec from sklearn.metrics import accuracy_score from timm.data import resolve_data_config @@ -201,13 +201,9 @@ def main( aten_dialect: ExportedProgram = export(quantized_model, example_args) - # Convert to edge dialect - edge_program: EdgeProgramManager = to_edge(aten_dialect) - to_be_lowered_module = edge_program.exported_program() - - # Lower the module to the backend with a custom partitioner + # Convert to edge dialect and lower the module to the backend with a custom partitioner compile_spec = [CompileSpec("device", device.encode())] - lowered_module = edge_program.to_backend(OpenvinoPartitioner(compile_spec)) + lowered_module: EdgeProgramManager = to_edge_transform_and_lower(aten_dialect, partitioner=[OpenvinoPartitioner(compile_spec),]) # Apply backend-specific passes exec_prog = lowered_module.to_executorch(config=executorch.exir.ExecutorchBackendConfig()) diff --git a/examples/openvino/executor_runner/openvino_executor_runner.cpp b/examples/openvino/executor_runner/openvino_executor_runner.cpp index c3922c793a3..17cc91ba3e9 100644 --- a/examples/openvino/executor_runner/openvino_executor_runner.cpp +++ b/examples/openvino/executor_runner/openvino_executor_runner.cpp @@ -101,7 +101,7 @@ std::vector> get_inputs_paths(const char *input_list_path) { size_t idx = 0; - auto split = [](std::string s, std::string delimiter) { + auto split_and_add_prefix = [](std::string s, std::string delimiter, std::string prefix = "") { size_t pos_start = 0, pos_end, delim_len = delimiter.length(); std::string token; std::vector res; @@ -109,9 +109,9 @@ get_inputs_paths(const char *input_list_path) { while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) { token = s.substr(pos_start, pos_end - pos_start); pos_start = pos_end + delim_len; - res.push_back(token); + res.push_back(prefix + token); } - res.push_back(s.substr(pos_start)); + res.push_back(prefix + s.substr(pos_start)); return res; }; @@ -121,10 +121,15 @@ get_inputs_paths(const char *input_list_path) { if (!input_list.is_open()) { ET_CHECK_MSG(false, "Failed to read input list file: %s", input_list_path); } + std::string inputs_dir = ""; + size_t last_pos = std::string(input_list_path).rfind('/'); + if (last_pos != std::string::npos) { + inputs_dir = std::string(input_list_path).substr(0, last_pos+1); + } std::string file_path; auto retval = std::vector>(); while (std::getline(input_list, file_path)) { - auto input_files = split(file_path, " "); + auto input_files = split_and_add_prefix(file_path, " ", inputs_dir); if (input_files.size() == 0) { break; }