From 70a3b2a04ca08162f22597764e6c209d2bb477e5 Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Wed, 12 Feb 2025 14:24:36 -0800 Subject: [PATCH 01/12] Use to_edge_transform_and_lower in aot_openvino_compiler --- examples/openvino/aot/aot_openvino_compiler.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/examples/openvino/aot/aot_openvino_compiler.py b/examples/openvino/aot/aot_openvino_compiler.py index 4674fbbd755..242a4fa1532 100644 --- a/examples/openvino/aot/aot_openvino_compiler.py +++ b/examples/openvino/aot/aot_openvino_compiler.py @@ -12,7 +12,7 @@ from executorch.exir.backend.backend_details import CompileSpec from executorch.backends.openvino.preprocess import OpenvinoBackend from executorch.backends.openvino.partitioner import OpenvinoPartitioner -from executorch.exir import EdgeProgramManager, to_edge +from executorch.exir import EdgeProgramManager, to_edge_transform_and_lower from torch.export import export, ExportedProgram from torch.export.exported_program import ExportedProgram import argparse @@ -47,13 +47,9 @@ def main(suite: str, model_name: str, input_shape, device: str): # Export to aten dialect using torch.export aten_dialect: ExportedProgram = export(model, example_args) - # Convert to edge dialect - edge_program: EdgeProgramManager = to_edge(aten_dialect) - to_be_lowered_module = edge_program.exported_program() - - # Lower the module to the backend with a custom partitioner + # Convert to edge dialect and lower the module to the backend with a custom partitioner compile_spec = [CompileSpec("device", device.encode())] - lowered_module = edge_program.to_backend(OpenvinoPartitioner(compile_spec)) + lowered_module: EdgeProgramManager = to_edge_transform_and_lower(aten_dialect, partitioner=[OpenvinoPartitioner(compile_spec),]) # Apply backend-specific passes exec_prog = lowered_module.to_executorch(config=executorch.exir.ExecutorchBackendConfig()) From 5ea37c7ef5b7268277e11cb9d07dad447b252e37 Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Wed, 12 Feb 2025 14:26:18 -0800 Subject: [PATCH 02/12] Fix input tensor file reading bug --- .../executor_runner/openvino_executor_runner.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/openvino/executor_runner/openvino_executor_runner.cpp b/examples/openvino/executor_runner/openvino_executor_runner.cpp index 7615b63649a..1c58a49fbb0 100644 --- a/examples/openvino/executor_runner/openvino_executor_runner.cpp +++ b/examples/openvino/executor_runner/openvino_executor_runner.cpp @@ -175,6 +175,11 @@ int main(int argc, char** argv) { // iterate each raw input tensor file to read values std::ifstream input_list(input_list_path); if (input_list.is_open()) { + std::string inputs_dir = ""; + size_t last_pos = std::string(input_list_path).rfind('/'); + if (last_pos != std::string::npos) { + inputs_dir = std::string(input_list_path).substr(0, last_pos+1); + } size_t num_inputs = method->inputs_size(); std::string file_path; while (std::getline(input_list, file_path)) { @@ -188,7 +193,12 @@ int main(int argc, char** argv) { method_meta.input_tensor_meta(input_index); auto input_data_ptr = inputs[input_index].toTensor().data_ptr(); - std::ifstream fin(input_files[input_index], std::ios::binary); + std::ifstream fin(inputs_dir+input_files[input_index], std::ios::binary); + if (!(fin.good())) { + ET_CHECK_MSG(false, + "Failed to read input tensor file: %s", + inputs_dir+input_files[input_index]); + } fin.seekg(0, fin.end); size_t file_size = fin.tellg(); From 6beadb3c63ee7574abaa3d8c1e7965d84d761de8 Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Wed, 12 Feb 2025 14:27:16 -0800 Subject: [PATCH 03/12] Update unit tests --- backends/openvino/tests/ops/base_openvino_op_test.py | 3 +-- backends/openvino/tests/test_openvino_delegate.py | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index a51b99e8eca..88fafc25fe4 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -97,13 +97,12 @@ def execute_layer_test( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, - cwd=tmp_dir, ) stdout_str = proc.stdout.decode('utf-8') # Check if execution completed successfully - self.assertIn("Model executed successfully.", stdout_str) + self.assertTrue(proc.returncode == 0) # Read the outputs from the temporary files output_dir = f"{tmp_dir}/outputs" diff --git a/backends/openvino/tests/test_openvino_delegate.py b/backends/openvino/tests/test_openvino_delegate.py index eaabcf2603b..89763d1d960 100644 --- a/backends/openvino/tests/test_openvino_delegate.py +++ b/backends/openvino/tests/test_openvino_delegate.py @@ -71,4 +71,8 @@ def parse_arguments(): # Discover all existing op tests in "ops" folder suite = loader.discover(test_params['test_type'], pattern=test_params['pattern']) # Start running tests - unittest.TextTestRunner().run(suite) + result = unittest.TextTestRunner().run(suite) + if result.wasSuccessful(): + print("OpenVINO backend tests completed successfully") + else: + print("OpenVINO backend tests completed with failures") From 0c07b701aca0871cb3db455eb5ef7363042364d9 Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Wed, 12 Feb 2025 15:59:06 -0800 Subject: [PATCH 04/12] temp changes for debugging --- .../tests/ops/base_openvino_op_test.py | 4 +++- .../openvino_executor_runner.cpp | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index 88fafc25fe4..57b58c4ba32 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -77,7 +77,7 @@ def execute_layer_test( exec_prog.write_to_file(file) # Save inputs into a temporary file - self.generate_inputs(tmp_dir, "input_list.txt", [sample_inputs], input_list) + #self.generate_inputs(tmp_dir, "input_list.txt", [sample_inputs], input_list) self.make_output_dir(output_dir) # Start a subprocess to execute model with openvino_executor_runner @@ -100,6 +100,8 @@ def execute_layer_test( ) stdout_str = proc.stdout.decode('utf-8') + print("STDOUT:") + print(stdout_str) # Check if execution completed successfully self.assertTrue(proc.returncode == 0) diff --git a/examples/openvino/executor_runner/openvino_executor_runner.cpp b/examples/openvino/executor_runner/openvino_executor_runner.cpp index 1c58a49fbb0..a9cf4106149 100644 --- a/examples/openvino/executor_runner/openvino_executor_runner.cpp +++ b/examples/openvino/executor_runner/openvino_executor_runner.cpp @@ -80,6 +80,7 @@ int main(int argc, char** argv) { // Load the model using FileDataLoader Result loader = FileDataLoader::from(model_path); + std::cout << "AAA - 1" << std::endl; ET_CHECK_MSG( loader.ok(), "FileDataLoader::from() failed: 0x%" PRIx32, @@ -88,22 +89,27 @@ int main(int argc, char** argv) { // Load the program from the loaded model Result program = Program::load(&loader.get()); if (!program.ok()) { + std::cout << "AAA - 2" << std::endl; ET_LOG(Error, "Failed to parse model file %s", model_path); return 1; } + std::cout << "AAA - 3" << std::endl; ET_LOG(Info, "Model file %s is loaded.", model_path); // Retrieve the method name from the program (assumes the first method is used) const char* method_name = nullptr; { const auto method_name_result = program->get_method_name(0); + std::cout << "AAA - 4" << std::endl; ET_CHECK_MSG(method_name_result.ok(), "Program has no methods"); method_name = *method_name_result; } + std::cout << "AAA - 5" << std::endl; ET_LOG(Info, "Using method %s", method_name); // Retrieve metadata about the method Result method_meta = program->method_meta(method_name); + std::cout << "AAA - 6" << std::endl; ET_CHECK_MSG( method_meta.ok(), "Failed to get method_meta for %s: 0x%" PRIx32, @@ -121,6 +127,7 @@ int main(int argc, char** argv) { for (size_t id = 0; id < num_memory_planned_buffers; ++id) { size_t buffer_size = static_cast(method_meta->memory_planned_buffer_size(id).get()); + std::cout << "AAA - 7" << std::endl; ET_LOG(Info, "Setting up planned buffer %zu, size %zu.", id, buffer_size); planned_buffers.push_back(std::make_unique(buffer_size)); planned_spans.push_back({planned_buffers.back().get(), buffer_size}); @@ -133,15 +140,18 @@ int main(int argc, char** argv) { // Load the method into the program Result method = program->load_method(method_name, &memory_manager); + std::cout << "AAA - 8" << std::endl; ET_CHECK_MSG( method.ok(), "Loading of method %s failed with status 0x%" PRIx32, method_name, static_cast(method.error())); + std::cout << "AAA - 9" << std::endl; ET_LOG(Info, "Method loaded."); // Prepare the input tensors for the method auto inputs = prepare_input_tensors(*method); + std::cout << "AAA - 10" << std::endl; ET_CHECK_MSG( inputs.ok(), "Could not prepare inputs: 0x%" PRIx32, @@ -150,11 +160,14 @@ int main(int argc, char** argv) { // If the input path list is provided, read input tensors from the files if (!(FLAGS_input_list_path.empty())) { const char* input_list_path = FLAGS_input_list_path.c_str(); + std::cout << "AAA - 11" << std::endl; ET_LOG(Info, "Loading input tensors from the list provided in %s.", input_list_path); Error status = Error::Ok; std::vector inputs(method->inputs_size()); + std::cout << "AAA - 12" << std::endl; ET_LOG(Info, "%zu inputs: ", inputs.size()); status = method->get_inputs(inputs.data(), inputs.size()); + std::cout << "AAA - 13" << std::endl; ET_CHECK(status == Error::Ok); auto split = [](std::string s, std::string delimiter) { @@ -195,6 +208,7 @@ int main(int argc, char** argv) { std::ifstream fin(inputs_dir+input_files[input_index], std::ios::binary); if (!(fin.good())) { + std::cout << "AAA - 14" << std::endl; ET_CHECK_MSG(false, "Failed to read input tensor file: %s", inputs_dir+input_files[input_index]); @@ -202,6 +216,7 @@ int main(int argc, char** argv) { fin.seekg(0, fin.end); size_t file_size = fin.tellg(); + std::cout << "AAA - 15" << std::endl; ET_CHECK_MSG( file_size == tensor_meta->nbytes(), "Input(%d) size mismatch. file bytes: %zu, tensor bytes: %zu", @@ -217,11 +232,13 @@ int main(int argc, char** argv) { } } } else { + std::cout << "AAA - 16" << std::endl; ET_CHECK_MSG(false, "Failed to read input list file: %s", input_list_path); } } + std::cout << "AAA - 17" << std::endl; ET_LOG(Info, "Inputs prepared."); // Measure execution time for inference @@ -236,29 +253,35 @@ int main(int argc, char** argv) { .count() / 1000.0; // Log execution time and average time per iteration + std::cout << "AAA - 18" << std::endl; ET_LOG( Info, "%d inference took %f ms, avg %f ms", num_iterations, elapsed_time, elapsed_time / static_cast(num_iterations)); + std::cout << "AAA - 19" << std::endl; ET_CHECK_MSG( status == Error::Ok, "Execution of method %s failed with status 0x%" PRIx32, method_name, static_cast(status)); + std::cout << "AAA - 20" << std::endl; ET_LOG(Info, "Model executed successfully."); // Retrieve and print the method outputs std::vector outputs(method->outputs_size()); + std::cout << "AAA - 21" << std::endl; ET_LOG(Info, "%zu Number of outputs: ", outputs.size()); status = method->get_outputs(outputs.data(), outputs.size()); + std::cout << "AAA - 22" << std::endl; ET_CHECK(status == Error::Ok); // If output folder path is provided, save output tensors // into raw tensor files. if (!(FLAGS_output_folder_path.empty())) { const char* output_folder_path = FLAGS_output_folder_path.c_str(); + std::cout << "AAA - 23" << std::endl; ET_LOG(Info, "Saving output tensors into the output folder: %s.", output_folder_path); for (size_t output_index = 0; output_index < method->outputs_size(); output_index++) { @@ -271,6 +294,7 @@ int main(int argc, char** argv) { fout.close(); } } + std::cout << "AAA - 24" << std::endl; return 0; } From b0862027bdc74f0986b98e726c2d19cbab7227af Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Wed, 12 Feb 2025 17:28:45 -0800 Subject: [PATCH 05/12] Initial document for openvino backend tests --- backends/openvino/tests/README.md | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 backends/openvino/tests/README.md diff --git a/backends/openvino/tests/README.md b/backends/openvino/tests/README.md new file mode 100644 index 00000000000..964357c58e0 --- /dev/null +++ b/backends/openvino/tests/README.md @@ -0,0 +1,52 @@ +# Unit Tests for OpenVINO Backend + +## Directory Structure + +Below is the layout of the `backends/openvino/tests` directory, which includes the necessary files for the example applications: + +``` +backends/openvino/tests +├── ops # Directory with base op test script and individual op tests. + ├── base_openvino_op_test.py # Script which contains the base class for all op tests. + └── test_.py # Individual op tests scripts. +├── models # Directory with model test scripts. + └── test_classification.py # Test script for classification models. +├── README.md # Documentation for unit tests (this file) +└── test_openvino_delegate.py # Script to execute unit tests. +``` + +## Executing Unit Tests + +### Prerequisites + +Before you begin, refer to instructions provided in [OpenVINO Backend for ExecuTorch](./README.md) to install openvino and setup executorch environment. +Once openvino is installed and executorch environment is set, refer to [asdfasdf](asdfasdf) to build openvino_example_runner. + +### Usage + +test_openvino_delegate.py allows to run op or model tests for openvino backend. + +### **Arguments** +- **`--build_folder`** (required): + Path to cmake binary directory. (Refer to [asdf](asdf)) + Examples: + - `../../../cmake-openvino-out` (Relative path from `backends/openvino/tests` directory) + - `/cmake-openvino-out` (Absolute path to the default build folder) + +- **`--test_type`** (optional): + Type of the tests to run. + Supported values: + - `ops` (default) + - `models` + +- **`--pattern`** (optional): + Pattern to match test files. Provide complete file name to run individual tests. The default value is `test_*.py` + Examples: + - `test_convolution.py` (Assuming `--test_type` parameter is provided as `ops`, this will run only convolution tests) + - `test_add*.py` (Assuming `--test_type` parameter is provided as `ops`, this will run add and addmm op tests) + +- **`--device`** (optional): + Target device to compile and run tests. Default is `CPU`. + Examples: `CPU`, `GPU` + + From cb91596c511dedeb3754896e22a36783eeb1546d Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Wed, 12 Feb 2025 17:38:39 -0800 Subject: [PATCH 06/12] Update openvino backend test documentation --- backends/openvino/tests/README.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/backends/openvino/tests/README.md b/backends/openvino/tests/README.md index 964357c58e0..61e959f2db1 100644 --- a/backends/openvino/tests/README.md +++ b/backends/openvino/tests/README.md @@ -19,8 +19,8 @@ backends/openvino/tests ### Prerequisites -Before you begin, refer to instructions provided in [OpenVINO Backend for ExecuTorch](./README.md) to install openvino and setup executorch environment. -Once openvino is installed and executorch environment is set, refer to [asdfasdf](asdfasdf) to build openvino_example_runner. +Before you begin, refer to instructions provided in [OpenVINO Backend for ExecuTorch](../README.md) to install openvino and setup executorch environment. +Once openvino is installed and executorch environment is set, refer to [OpenVINO Backend Examples](../../../examples/openvino/README.md) to build openvino_example_runner. ### Usage @@ -28,7 +28,7 @@ test_openvino_delegate.py allows to run op or model tests for openvino backend. ### **Arguments** - **`--build_folder`** (required): - Path to cmake binary directory. (Refer to [asdf](asdf)) + Path to cmake binary directory. (Refer to [OpenVINO Backend Examples](../../../examples/openvino/README.md)) Examples: - `../../../cmake-openvino-out` (Relative path from `backends/openvino/tests` directory) - `/cmake-openvino-out` (Absolute path to the default build folder) @@ -50,3 +50,18 @@ test_openvino_delegate.py allows to run op or model tests for openvino backend. Examples: `CPU`, `GPU` +## **Examples** + +### Execute Tests for All Ops on CPU +```bash +python test_openvino_delegate.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops +``` + +### Execute Convolution Op Tests on CPU +```bash +python test_openvino_delegate.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops --pattern test_convolution.py +``` + +### Execute Tests for all Models on GPU +```bash +python test_openvino_delegate.py --build_folder ../../../cmake-openvino-out --device GPU --test_type models From d7567277978ab4b49461e72e7590a11bf0686be0 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Wed, 12 Feb 2025 17:39:46 -0800 Subject: [PATCH 07/12] Update README.md --- backends/openvino/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/openvino/tests/README.md b/backends/openvino/tests/README.md index 61e959f2db1..e573408d197 100644 --- a/backends/openvino/tests/README.md +++ b/backends/openvino/tests/README.md @@ -24,7 +24,7 @@ Once openvino is installed and executorch environment is set, refer to [OpenVINO ### Usage -test_openvino_delegate.py allows to run op or model tests for openvino backend. +`test_openvino_delegate.py` allows to run op or model tests for openvino backend. ### **Arguments** - **`--build_folder`** (required): From 09b2dbdd950da8ce777ffd15ed1ef8fd50f203dd Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Wed, 12 Feb 2025 18:04:39 -0800 Subject: [PATCH 08/12] Removed debugging lines --- .../tests/ops/base_openvino_op_test.py | 4 ---- .../openvino_executor_runner.cpp | 24 ------------------- 2 files changed, 28 deletions(-) diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index 57b58c4ba32..5ee04734918 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -99,10 +99,6 @@ def execute_layer_test( env=env, ) - stdout_str = proc.stdout.decode('utf-8') - print("STDOUT:") - print(stdout_str) - # Check if execution completed successfully self.assertTrue(proc.returncode == 0) diff --git a/examples/openvino/executor_runner/openvino_executor_runner.cpp b/examples/openvino/executor_runner/openvino_executor_runner.cpp index a9cf4106149..1c58a49fbb0 100644 --- a/examples/openvino/executor_runner/openvino_executor_runner.cpp +++ b/examples/openvino/executor_runner/openvino_executor_runner.cpp @@ -80,7 +80,6 @@ int main(int argc, char** argv) { // Load the model using FileDataLoader Result loader = FileDataLoader::from(model_path); - std::cout << "AAA - 1" << std::endl; ET_CHECK_MSG( loader.ok(), "FileDataLoader::from() failed: 0x%" PRIx32, @@ -89,27 +88,22 @@ int main(int argc, char** argv) { // Load the program from the loaded model Result program = Program::load(&loader.get()); if (!program.ok()) { - std::cout << "AAA - 2" << std::endl; ET_LOG(Error, "Failed to parse model file %s", model_path); return 1; } - std::cout << "AAA - 3" << std::endl; ET_LOG(Info, "Model file %s is loaded.", model_path); // Retrieve the method name from the program (assumes the first method is used) const char* method_name = nullptr; { const auto method_name_result = program->get_method_name(0); - std::cout << "AAA - 4" << std::endl; ET_CHECK_MSG(method_name_result.ok(), "Program has no methods"); method_name = *method_name_result; } - std::cout << "AAA - 5" << std::endl; ET_LOG(Info, "Using method %s", method_name); // Retrieve metadata about the method Result method_meta = program->method_meta(method_name); - std::cout << "AAA - 6" << std::endl; ET_CHECK_MSG( method_meta.ok(), "Failed to get method_meta for %s: 0x%" PRIx32, @@ -127,7 +121,6 @@ int main(int argc, char** argv) { for (size_t id = 0; id < num_memory_planned_buffers; ++id) { size_t buffer_size = static_cast(method_meta->memory_planned_buffer_size(id).get()); - std::cout << "AAA - 7" << std::endl; ET_LOG(Info, "Setting up planned buffer %zu, size %zu.", id, buffer_size); planned_buffers.push_back(std::make_unique(buffer_size)); planned_spans.push_back({planned_buffers.back().get(), buffer_size}); @@ -140,18 +133,15 @@ int main(int argc, char** argv) { // Load the method into the program Result method = program->load_method(method_name, &memory_manager); - std::cout << "AAA - 8" << std::endl; ET_CHECK_MSG( method.ok(), "Loading of method %s failed with status 0x%" PRIx32, method_name, static_cast(method.error())); - std::cout << "AAA - 9" << std::endl; ET_LOG(Info, "Method loaded."); // Prepare the input tensors for the method auto inputs = prepare_input_tensors(*method); - std::cout << "AAA - 10" << std::endl; ET_CHECK_MSG( inputs.ok(), "Could not prepare inputs: 0x%" PRIx32, @@ -160,14 +150,11 @@ int main(int argc, char** argv) { // If the input path list is provided, read input tensors from the files if (!(FLAGS_input_list_path.empty())) { const char* input_list_path = FLAGS_input_list_path.c_str(); - std::cout << "AAA - 11" << std::endl; ET_LOG(Info, "Loading input tensors from the list provided in %s.", input_list_path); Error status = Error::Ok; std::vector inputs(method->inputs_size()); - std::cout << "AAA - 12" << std::endl; ET_LOG(Info, "%zu inputs: ", inputs.size()); status = method->get_inputs(inputs.data(), inputs.size()); - std::cout << "AAA - 13" << std::endl; ET_CHECK(status == Error::Ok); auto split = [](std::string s, std::string delimiter) { @@ -208,7 +195,6 @@ int main(int argc, char** argv) { std::ifstream fin(inputs_dir+input_files[input_index], std::ios::binary); if (!(fin.good())) { - std::cout << "AAA - 14" << std::endl; ET_CHECK_MSG(false, "Failed to read input tensor file: %s", inputs_dir+input_files[input_index]); @@ -216,7 +202,6 @@ int main(int argc, char** argv) { fin.seekg(0, fin.end); size_t file_size = fin.tellg(); - std::cout << "AAA - 15" << std::endl; ET_CHECK_MSG( file_size == tensor_meta->nbytes(), "Input(%d) size mismatch. file bytes: %zu, tensor bytes: %zu", @@ -232,13 +217,11 @@ int main(int argc, char** argv) { } } } else { - std::cout << "AAA - 16" << std::endl; ET_CHECK_MSG(false, "Failed to read input list file: %s", input_list_path); } } - std::cout << "AAA - 17" << std::endl; ET_LOG(Info, "Inputs prepared."); // Measure execution time for inference @@ -253,35 +236,29 @@ int main(int argc, char** argv) { .count() / 1000.0; // Log execution time and average time per iteration - std::cout << "AAA - 18" << std::endl; ET_LOG( Info, "%d inference took %f ms, avg %f ms", num_iterations, elapsed_time, elapsed_time / static_cast(num_iterations)); - std::cout << "AAA - 19" << std::endl; ET_CHECK_MSG( status == Error::Ok, "Execution of method %s failed with status 0x%" PRIx32, method_name, static_cast(status)); - std::cout << "AAA - 20" << std::endl; ET_LOG(Info, "Model executed successfully."); // Retrieve and print the method outputs std::vector outputs(method->outputs_size()); - std::cout << "AAA - 21" << std::endl; ET_LOG(Info, "%zu Number of outputs: ", outputs.size()); status = method->get_outputs(outputs.data(), outputs.size()); - std::cout << "AAA - 22" << std::endl; ET_CHECK(status == Error::Ok); // If output folder path is provided, save output tensors // into raw tensor files. if (!(FLAGS_output_folder_path.empty())) { const char* output_folder_path = FLAGS_output_folder_path.c_str(); - std::cout << "AAA - 23" << std::endl; ET_LOG(Info, "Saving output tensors into the output folder: %s.", output_folder_path); for (size_t output_index = 0; output_index < method->outputs_size(); output_index++) { @@ -294,7 +271,6 @@ int main(int argc, char** argv) { fout.close(); } } - std::cout << "AAA - 24" << std::endl; return 0; } From 49530338ca23c7434fc91460932d6ce36cc00f7a Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Wed, 12 Feb 2025 18:08:55 -0800 Subject: [PATCH 09/12] Removed comment which was added for debugging --- backends/openvino/tests/ops/base_openvino_op_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index 5ee04734918..46c0b63fc37 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -77,7 +77,7 @@ def execute_layer_test( exec_prog.write_to_file(file) # Save inputs into a temporary file - #self.generate_inputs(tmp_dir, "input_list.txt", [sample_inputs], input_list) + self.generate_inputs(tmp_dir, "input_list.txt", [sample_inputs], input_list) self.make_output_dir(output_dir) # Start a subprocess to execute model with openvino_executor_runner From a9800995235770af8f13231c50777c5a022f7251 Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Fri, 14 Feb 2025 16:23:26 -0800 Subject: [PATCH 10/12] Fix for input file path bug --- .../executor_runner/openvino_executor_runner.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/openvino/executor_runner/openvino_executor_runner.cpp b/examples/openvino/executor_runner/openvino_executor_runner.cpp index c3922c793a3..17cc91ba3e9 100644 --- a/examples/openvino/executor_runner/openvino_executor_runner.cpp +++ b/examples/openvino/executor_runner/openvino_executor_runner.cpp @@ -101,7 +101,7 @@ std::vector> get_inputs_paths(const char *input_list_path) { size_t idx = 0; - auto split = [](std::string s, std::string delimiter) { + auto split_and_add_prefix = [](std::string s, std::string delimiter, std::string prefix = "") { size_t pos_start = 0, pos_end, delim_len = delimiter.length(); std::string token; std::vector res; @@ -109,9 +109,9 @@ get_inputs_paths(const char *input_list_path) { while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) { token = s.substr(pos_start, pos_end - pos_start); pos_start = pos_end + delim_len; - res.push_back(token); + res.push_back(prefix + token); } - res.push_back(s.substr(pos_start)); + res.push_back(prefix + s.substr(pos_start)); return res; }; @@ -121,10 +121,15 @@ get_inputs_paths(const char *input_list_path) { if (!input_list.is_open()) { ET_CHECK_MSG(false, "Failed to read input list file: %s", input_list_path); } + std::string inputs_dir = ""; + size_t last_pos = std::string(input_list_path).rfind('/'); + if (last_pos != std::string::npos) { + inputs_dir = std::string(input_list_path).substr(0, last_pos+1); + } std::string file_path; auto retval = std::vector>(); while (std::getline(input_list, file_path)) { - auto input_files = split(file_path, " "); + auto input_files = split_and_add_prefix(file_path, " ", inputs_dir); if (input_files.size() == 0) { break; } From 209132b586950d6078c9aaf698a47b6c9cf47b39 Mon Sep 17 00:00:00 2001 From: Cavus Mustafa Date: Fri, 14 Feb 2025 16:37:30 -0800 Subject: [PATCH 11/12] Test script name change --- .../openvino/tests/{test_openvino_delegate.py => test_runner.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename backends/openvino/tests/{test_openvino_delegate.py => test_runner.py} (100%) diff --git a/backends/openvino/tests/test_openvino_delegate.py b/backends/openvino/tests/test_runner.py similarity index 100% rename from backends/openvino/tests/test_openvino_delegate.py rename to backends/openvino/tests/test_runner.py From 4a27c36e1eeb8ed60b4e5cfed4d27c749618f793 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Fri, 14 Feb 2025 16:18:42 -0800 Subject: [PATCH 12/12] Update README.md --- backends/openvino/tests/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backends/openvino/tests/README.md b/backends/openvino/tests/README.md index e573408d197..3ad109274f5 100644 --- a/backends/openvino/tests/README.md +++ b/backends/openvino/tests/README.md @@ -12,7 +12,7 @@ backends/openvino/tests ├── models # Directory with model test scripts. └── test_classification.py # Test script for classification models. ├── README.md # Documentation for unit tests (this file) -└── test_openvino_delegate.py # Script to execute unit tests. +└── test_runner.py # Script to execute unit tests. ``` ## Executing Unit Tests @@ -24,7 +24,7 @@ Once openvino is installed and executorch environment is set, refer to [OpenVINO ### Usage -`test_openvino_delegate.py` allows to run op or model tests for openvino backend. +`test_runner.py` allows to run op or model tests for openvino backend. ### **Arguments** - **`--build_folder`** (required): @@ -54,14 +54,14 @@ Once openvino is installed and executorch environment is set, refer to [OpenVINO ### Execute Tests for All Ops on CPU ```bash -python test_openvino_delegate.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops +python test_runner.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops ``` ### Execute Convolution Op Tests on CPU ```bash -python test_openvino_delegate.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops --pattern test_convolution.py +python test_runner.py --build_folder ../../../cmake-openvino-out --device CPU --test_type ops --pattern test_convolution.py ``` ### Execute Tests for all Models on GPU ```bash -python test_openvino_delegate.py --build_folder ../../../cmake-openvino-out --device GPU --test_type models +python test_runner.py --build_folder ../../../cmake-openvino-out --device GPU --test_type models