From a554e5e5a8a9701ff7a3ae694b1e7f8c18938409 Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 23 Sep 2024 14:32:23 -0700 Subject: [PATCH 01/32] fix: caching lookup to behave correctly when inputs/output mapping are changed. --- .../openvino/backends/basic_backend.cc | 76 +++++-------------- .../openvino/backends/basic_backend.h | 4 +- 2 files changed, 19 insertions(+), 61 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 1f9c61780f27a..0270de51251a7 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -303,33 +303,18 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque FillInputBlob(std::move(graph_input_blob), batch_slice_idx, std::move(input_name), context, subgraph_context_); } else { auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_key; - ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_key = it->second; - } else { - // Does this make sense for both types of allocators? + ort_tensor_key_t ort_tensor_key{input_name}; + auto it = ort_ov_tensor_map.find(ort_tensor_key); + if (it != ort_ov_tensor_map.end() || it->second.ort_ptr != tensor.GetTensorRawData()) { + ov_tensor_data_t ov_tensor_data; auto input = graph_input_info.at(input_idx); - if (allocator_name == OpenVINO_RT_NPU) { - ov_tensor_key.copy_needed = false; - ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), - (void*)tensor.GetTensorRawData()); - } else { - ov_tensor_key.copy_needed = true; - ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape()); - } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); - - if (ov_tensor_key.copy_needed) { - const char* ort_tensor_data = tensor.GetTensorData(); - size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; - std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); - } + ov_tensor_data.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), + (void*)tensor.GetTensorRawData()); + ov_tensor_data.ort_ptr = tensor.GetTensorRawData(); + ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; try { - infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + infer_request->SetTensor(input_name, ov_tensor_data.tensor_ptr); } catch (const char* msg) { ORT_THROW(msg); } @@ -362,23 +347,15 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque infer_request, output_name, subgraph_context_.output_names); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); - - ov_tensor_data_t ov_tensor_data; - ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; - } else { + ort_tensor_key_t ort_tensor_key{output_name}; + const auto& it = ort_ov_tensor_map.find(ort_tensor_key); + if (it != ort_ov_tensor_map.end() || it->second.ort_ptr != tensor.GetTensorRawData()) { + ov_tensor_data_t ov_tensor_data; auto output = graph_output_info.at(output_idx); - if (allocator_name == OpenVINO_RT_NPU) { - ov_tensor_data.copy_needed = false; - ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), - (void*)tensor.GetTensorRawData()); - } else { - ov_tensor_data.copy_needed = true; - ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape()); - } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); + ov_tensor_data.ort_ptr = tensor.GetTensorRawData(); + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), + (void*)tensor.GetTensorRawData()); + ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; try { infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); @@ -556,25 +533,6 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe size_t batch_slice = 0; FillOutputBlob(std::move(graph_output_blob), output_tensor, batch_slice); } - } else { - size_t batch_size = 1; - Ort::UnownedValue output_tensor = - GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); - auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_data; - ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; - } else { - ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); - } - - if (ov_tensor_data.copy_needed) { - auto ort_tensor_data = output_tensor.GetTensorMutableData(); - size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; - std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); - } } } diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index cd69e88f994b9..875e5941a4f14 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -23,7 +23,7 @@ namespace openvino_ep { struct ov_tensor_data_t { OVTensorPtr tensor_ptr; - bool copy_needed; + const void *ort_ptr; }; class InferRequestsQueue; @@ -67,7 +67,7 @@ class BasicBackend : public IBackend { OVRemoteContextPtr remote_context_; #endif - using ort_tensor_key_t = std::pair; + using ort_tensor_key_t = const std::string; std::map ort_ov_tensor_map; }; From f70b885d70a5bebe0f3d862de20ab43c2386fa66 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Tue, 24 Sep 2024 02:14:30 -0700 Subject: [PATCH 02/32] fix tensor caching --- onnxruntime/core/providers/openvino/backends/basic_backend.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 0270de51251a7..a8039255773fe 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -305,7 +305,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); ort_tensor_key_t ort_tensor_key{input_name}; auto it = ort_ov_tensor_map.find(ort_tensor_key); - if (it != ort_ov_tensor_map.end() || it->second.ort_ptr != tensor.GetTensorRawData()) { + if ((it == ort_ov_tensor_map.end()) || (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { ov_tensor_data_t ov_tensor_data; auto input = graph_input_info.at(input_idx); ov_tensor_data.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), @@ -349,7 +349,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque subgraph_context_.output_names); ort_tensor_key_t ort_tensor_key{output_name}; const auto& it = ort_ov_tensor_map.find(ort_tensor_key); - if (it != ort_ov_tensor_map.end() || it->second.ort_ptr != tensor.GetTensorRawData()) { + if ((it == ort_ov_tensor_map.end()) || (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { ov_tensor_data_t ov_tensor_data; auto output = graph_output_info.at(output_idx); ov_tensor_data.ort_ptr = tensor.GetTensorRawData(); From ce64f7aedfd2f392ec8dd155d23c7afea5c15549 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Tue, 24 Sep 2024 04:18:45 -0700 Subject: [PATCH 03/32] fix lint issues --- onnxruntime/core/providers/openvino/backends/basic_backend.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index 875e5941a4f14..12502a1d83c5d 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -23,7 +23,7 @@ namespace openvino_ep { struct ov_tensor_data_t { OVTensorPtr tensor_ptr; - const void *ort_ptr; + const void* ort_ptr; }; class InferRequestsQueue; From 086a04847943d4230d5c6d0403653e3d40993619 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Tue, 24 Sep 2024 23:27:23 -0700 Subject: [PATCH 04/32] fix lint issues --- .../core/providers/openvino/backends/basic_backend.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index a8039255773fe..71a02f076c8cc 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -305,11 +305,13 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); ort_tensor_key_t ort_tensor_key{input_name}; auto it = ort_ov_tensor_map.find(ort_tensor_key); - if ((it == ort_ov_tensor_map.end()) || (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { + if ((it == ort_ov_tensor_map.end()) || + (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { ov_tensor_data_t ov_tensor_data; auto input = graph_input_info.at(input_idx); ov_tensor_data.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), - (void*)tensor.GetTensorRawData()); + const_cast(tensor.GetTensorRawData())); + ov_tensor_data.ort_ptr = tensor.GetTensorRawData(); ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; @@ -349,12 +351,13 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque subgraph_context_.output_names); ort_tensor_key_t ort_tensor_key{output_name}; const auto& it = ort_ov_tensor_map.find(ort_tensor_key); - if ((it == ort_ov_tensor_map.end()) || (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { + if ((it == ort_ov_tensor_map.end()) || + (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { ov_tensor_data_t ov_tensor_data; auto output = graph_output_info.at(output_idx); ov_tensor_data.ort_ptr = tensor.GetTensorRawData(); ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), - (void*)tensor.GetTensorRawData()); + const_cast(tensor.GetTensorRawData())); ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; try { From d85d7bbe00a6b0a74f52e583ba451cf01e817c2e Mon Sep 17 00:00:00 2001 From: saurabh Date: Thu, 5 Sep 2024 18:30:37 +0530 Subject: [PATCH 05/32] Improvement in average inference latency for models running on OVEP NPU (#441) * Prototype shared memory allocator on Windows using OV-EP * Partially working allocator. Crashing on tensor destruction. Might have UMD exceptions. Needs further debug. Unknown if values are correct. * Hard code onnx perf to use RT NPU allocator for inputs * Fix allocation lookups coming from different level zero contexts * Page align OV allocation * Allocate input as WC * Only set tensors when they have changed. * Revert "Allocate input as WC" This reverts commit d43219f9b794d3ff86dd71df162932a44ab4ff42. * Hard code onnx perf to use RT NPU for outputs * Revert "Hard code onnx perf to use RT NPU for outputs" This reverts commit c1f3b3ec7caee7024b0ef887fa762d36001e0daa. * Hard code onnx perf to use RT NPU for outputs fixed * Fix onnx_perf_test app crash on tensor destroy * refactor: remove redundant ort_shape_to_ovshape lambda function * alocate buffer in NPU visible region from perf test application * remove redundant code * add command line parameter in perf test for using remote tensors * remove redundant code * remove redundant statements * fix crash during inference * remove redundant code * enable backward compatibility of remote tensor feature * Revert "enable backward compatibility of remote tensor feature" This reverts commit 1791b907651e3fca18d7257f97487e463a3303cb. * enable backward compatibility of remote tensor feature in OVEP --------- Co-authored-by: Javier E. Martinez Co-authored-by: Eric Crawford --- cmake/onnxruntime_providers_openvino.cmake | 2 +- .../providers/openvino/backends/basic_backend.cc | 1 - .../openvino/openvino_execution_provider.cc | 12 ++++++------ .../providers/openvino/openvino_execution_provider.h | 2 +- onnxruntime/core/providers/openvino/ov_allocator.cc | 2 +- onnxruntime/core/providers/openvino/ov_allocator.h | 2 +- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 2eb3611bae902..69805d60d4593 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -22,7 +22,7 @@ endif() if(OpenVINO_VERSION VERSION_GREATER_EQUAL 2024.4) - add_definitions(-DUSE_OVEP_NPU_MEMORY=1) + add_definitions(-DUSE_DEVICE_MEMORY=1) endif() if (WIN32) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 1f9c61780f27a..17901d0ae984c 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -515,7 +515,6 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe auto graph_output_info = exe_network_.Get().outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { - OVTensorPtr graph_output_blob; auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 08144651319cf..4f9764212f37d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,7 +10,7 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" -#ifdef USE_OVEP_NPU_MEMORY +#ifdef USE_DEVICE_MEMORY #include "core/providers/openvino/ov_allocator.h" #endif @@ -183,13 +183,13 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } -#ifdef USE_OVEP_NPU_MEMORY +#ifdef USE_DEVICE_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { - AllocatorCreationInfo npu_allocator_info{ - [this](OrtDevice::DeviceId device_id) { + AllocatorCreationInfo npu_allocator_info { + [this](OrtDevice::DeviceId device_id) { return std::make_unique(global_context_->ie_core.Get(), OrtDevice::NPU, device_id, OpenVINO_RT_NPU); - }, - 0, + }, + 0, }; // fill in allocator diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 8b1c62c607f6e..42a8368a57e29 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,7 +189,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } -#ifdef USE_OVEP_NPU_MEMORY +#ifdef USE_DEVICE_MEMORY std::vector CreatePreferredAllocators() override; #endif private: diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 6700244b754d8..48ed626981dfb 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_OVEP_NPU_MEMORY +#ifdef USE_DEVICE_MEMORY #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 083cfc4d5aed3..24d61734a3c0b 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_OVEP_NPU_MEMORY +#ifdef USE_DEVICE_MEMORY #pragma once #include "core/common/inlined_containers.h" From d455fe0234d9dbff149788dbe88174acee6155fb Mon Sep 17 00:00:00 2001 From: saurabh Date: Thu, 12 Sep 2024 00:13:54 +0530 Subject: [PATCH 06/32] Ovep release lnl 1.2.1 (#445) * fix debug build issue and lint issues * change naming for OVEP NPU specific macro * fix unit tests and lint issues --- cmake/onnxruntime_providers_openvino.cmake | 2 +- .../providers/openvino/backends/basic_backend.cc | 1 + .../openvino/openvino_execution_provider.cc | 12 ++++++------ .../providers/openvino/openvino_execution_provider.h | 2 +- onnxruntime/core/providers/openvino/ov_allocator.cc | 2 +- onnxruntime/core/providers/openvino/ov_allocator.h | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 69805d60d4593..2eb3611bae902 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -22,7 +22,7 @@ endif() if(OpenVINO_VERSION VERSION_GREATER_EQUAL 2024.4) - add_definitions(-DUSE_DEVICE_MEMORY=1) + add_definitions(-DUSE_OVEP_NPU_MEMORY=1) endif() if (WIN32) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 17901d0ae984c..1f9c61780f27a 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -515,6 +515,7 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe auto graph_output_info = exe_network_.Get().outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { + OVTensorPtr graph_output_blob; auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 4f9764212f37d..08144651319cf 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,7 +10,7 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #endif @@ -183,13 +183,13 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { - AllocatorCreationInfo npu_allocator_info { - [this](OrtDevice::DeviceId device_id) { + AllocatorCreationInfo npu_allocator_info{ + [this](OrtDevice::DeviceId device_id) { return std::make_unique(global_context_->ie_core.Get(), OrtDevice::NPU, device_id, OpenVINO_RT_NPU); - }, - 0, + }, + 0, }; // fill in allocator diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 42a8368a57e29..8b1c62c607f6e 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,7 +189,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY std::vector CreatePreferredAllocators() override; #endif private: diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 48ed626981dfb..6700244b754d8 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 24d61734a3c0b..083cfc4d5aed3 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #pragma once #include "core/common/inlined_containers.h" From 87fd31a124eaeaae9d3896367ad1e0bccc4ebce4 Mon Sep 17 00:00:00 2001 From: "Javier E. Martinez" Date: Fri, 13 Sep 2024 16:13:47 -0700 Subject: [PATCH 07/32] Refactor device memory implementation to make it more generic --- onnxruntime/test/perftest/ort_test_session.cc | 73 +++++++------------ onnxruntime/test/perftest/ort_test_session.h | 3 +- 2 files changed, 29 insertions(+), 47 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 3ed5eaee5a5f7..403ba89a83a43 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -39,13 +39,8 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); - if (!use_device_mem) { - auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), output_names_raw_ptr.size()); - } else { - session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); - } + session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration duration_seconds = end - start; @@ -857,10 +852,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " "should be a boolean i.e. true or false. Default value is false.\n"); } - } else if (key == "use_device_mem") { - if (value == "true" || value == "True") { - use_device_mem = true; - } + } else if (key == "device_memory_name") { + device_memory_name_ = std::move(value); } else { ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } @@ -905,25 +898,26 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); input_names_[i] = input_names_str_[i].c_str(); } - if (use_device_mem) { - Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + auto transform_fcn = std::function(); + if (device_memory_name_.empty()) { + transform_fcn = [](int64_t input) { return input; }; + } else { + Ort::MemoryInfo memory_info = Ort::MemoryInfo(device_memory_name_.data(), OrtArenaAllocator, 0, OrtMemTypeCPUOutput); custom_allocator_ = std::make_unique(session_, memory_info); - for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { - Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + allocator_ = *custom_allocator_; + transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + } - std::vector output_shape = tensor_info.GetShape(); + for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { + Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - // free dimensions are treated as 1 if not overridden - for (int64_t& dim : output_shape) { - if (dim == -1) { - dim = 1; - } - } + // free dimensions are treated as 1 if not overridden + std::vector output_shape = tensor_info.GetShape(); + std::transform(output_shape.begin(), output_shape.end(), output_shape.begin(), transform_fcn); - outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), - output_shape.size(), tensor_info.GetElementType())); - } + outputs_.push_back(Ort::Value::CreateTensor(allocator_, output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); } } @@ -1013,29 +1007,16 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - if (!use_device_mem) { - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - } std::vector input_node_dim = tensor_info.GetShape(); // free dimensions are treated as 1 if not overridden - for (int64_t& dim : input_node_dim) { - if (dim == -1) { - dim = 1; - } - } - if (use_device_mem) { - Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(0, i, std::move(input_tensor)); - } else { - auto allocator = Ort::AllocatorWithDefaultOptions(); - Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(0, i, std::move(input_tensor)); - } + auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); + + Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(0, i, std::move(input_tensor)); } } return true; diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index e33041a2a0958..d59d90a5080b6 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -38,6 +38,7 @@ class OnnxRuntimeTestSession : public TestSession { std::mt19937 rand_engine_; std::uniform_int_distribution dist_; std::vector> test_inputs_; + OrtAllocator* allocator_ = Ort::AllocatorWithDefaultOptions(); std::unique_ptr custom_allocator_; std::vector outputs_; std::vector output_names_; @@ -48,7 +49,7 @@ class OnnxRuntimeTestSession : public TestSession { std::vector input_names_str_; const int input_length_; std::string provider_name_; - bool use_device_mem = false; + std::string device_memory_name_; // Device memory type name to use from the list in allocator.h }; } // namespace perftest From 48f02911a1b793bb53db56aa2739e8b575f271db Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Thu, 19 Sep 2024 22:55:35 -0700 Subject: [PATCH 08/32] fix lint issues --- onnxruntime/test/perftest/ort_test_session.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index d59d90a5080b6..7d5e46983ad41 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -49,7 +49,7 @@ class OnnxRuntimeTestSession : public TestSession { std::vector input_names_str_; const int input_length_; std::string provider_name_; - std::string device_memory_name_; // Device memory type name to use from the list in allocator.h + std::string device_memory_name_; // Device memory type name to use from the list in allocator.h }; } // namespace perftest From 03906a3b4bd83fc290b5bee087e81797c5ed74e6 Mon Sep 17 00:00:00 2001 From: sfatimar Date: Mon, 23 Sep 2024 17:39:37 +0530 Subject: [PATCH 09/32] Modified Create Options to pass config options to execution Provider --- .../openvino/openvino_provider_factory.cc | 109 ++++++------------ .../openvino_provider_factory_creator.h | 3 +- .../shared_library/provider_interfaces.h | 1 + .../shared_library/provider_wrappedtypes.h | 4 + .../core/session/provider_bridge_ort.cc | 49 ++------ onnxruntime/test/util/default_providers.cc | 5 +- .../test/util/include/default_providers.h | 2 +- 7 files changed, 60 insertions(+), 113 deletions(-) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 077ecc717502f..7072ab3660c92 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -13,9 +13,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { const char* cache_dir, const char* model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, - bool export_ep_ctx_blob, bool enable_qdq_optimizer, - bool disable_cpu_fallback, - bool so_epctx_embed_mode) + bool enable_qdq_optimizer, const ConfigOptions& config_options + ) : precision_(precision), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), @@ -24,13 +23,11 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { context_(context), enable_opencl_throttling_(enable_opencl_throttling), disable_dynamic_shapes_(disable_dynamic_shapes), - export_ep_ctx_blob_(export_ep_ctx_blob), enable_qdq_optimizer_(enable_qdq_optimizer), - disable_cpu_fallback_(disable_cpu_fallback), - so_epctx_embed_mode_(so_epctx_embed_mode) { - device_type_ = (device_type == nullptr) ? "" : device_type; - cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; - } + config_options_(config_options) { + device_type_ = (device_type == nullptr) ? "" : device_type; + cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; +} ~OpenVINOProviderFactory() override { } @@ -48,18 +45,37 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { void* context_; bool enable_opencl_throttling_; bool disable_dynamic_shapes_; - bool export_ep_ctx_blob_; bool enable_qdq_optimizer_; - bool disable_cpu_fallback_; - bool so_epctx_embed_mode_; + const ConfigOptions& config_options_; }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { + + bool so_disable_cpu_fallback = config_options_.GetConfigOrDefault("session.disable_cpu_ep_fallback", "0") == "1"; + bool so_export_ep_ctx_blob = config_options_.GetConfigOrDefault("ep.context_enable", "0") == "1"; + bool so_epctx_embed_mode = config_options_.GetConfigOrDefault("ep.context_embed_mode", "1") == "1"; + std::string so_cache_path = config_options_.GetConfigOrDefault("ep.context_file_path", "").c_str(); + + if (so_export_ep_ctx_blob && !so_cache_path.empty()) { + cache_dir_ = so_cache_path; + auto file_path = std::filesystem::path(cache_dir_); + // ep_context_file_path_ file extension must be .onnx + if (file_path.extension().generic_string() == ".onnx") { + // ep_context_file_path_ must be provided as a directory, create it if doesn't exist + auto parent_path = file_path.parent_path(); + if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && + !std::filesystem::create_directory(parent_path)) { + ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n"); + } + } else { + ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir_ + " \n"); + } + } + OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_, - disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_, - disable_cpu_fallback_, - so_epctx_embed_mode_); + disable_dynamic_shapes_, so_export_ep_ctx_blob, enable_qdq_optimizer_, + so_disable_cpu_fallback, so_epctx_embed_mode); return std::make_unique(info); } @@ -77,7 +93,11 @@ struct OpenVINO_Provider : Provider { void* GetInfo() override { return &g_info; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { - auto& provider_options_map = *reinterpret_cast(void_params); + typedef std::pair buffer_t; + const buffer_t* buffer = reinterpret_cast(void_params); + + auto& provider_options_map = *buffer->first; + const ConfigOptions& config_options = buffer->second; std::string device_type = ""; // [device_type]: Overrides the accelerator hardware type and precision // with these values at runtime. @@ -102,16 +122,11 @@ struct OpenVINO_Provider : Provider { // with this value at runtime. bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU // device (Reduces CPU Utilization when using GPU) - bool export_ep_ctx_blob = false; // Whether to export the pre-compiled blob as an EPContext model. void* context = nullptr; bool enable_qdq_optimizer = false; - bool disable_cpu_fallback = false; - - bool so_epctx_embed_mode = true; - if (provider_options_map.find("device_type") != provider_options_map.end()) { device_type = provider_options_map.at("device_type").c_str(); @@ -272,52 +287,6 @@ struct OpenVINO_Provider : Provider { } } } - if (provider_options_map.find("so_export_ep_ctx_blob") != provider_options_map.end()) { - bool_flag = provider_options_map.at("so_export_ep_ctx_blob"); - if (bool_flag == "true" || bool_flag == "True") - export_ep_ctx_blob = true; - else if (bool_flag == "false" || bool_flag == "False") - export_ep_ctx_blob = false; - bool_flag = ""; - } - - if (provider_options_map.find("disable_cpu_fallback") != provider_options_map.end()) { - bool_flag = provider_options_map.at("disable_cpu_fallback"); - if (bool_flag == "true" || bool_flag == "True") - disable_cpu_fallback = true; - else if (bool_flag == "false" || bool_flag == "False") - disable_cpu_fallback = false; - bool_flag = ""; - } - if (provider_options_map.find("so_epctx_embed_mode") != provider_options_map.end()) { - bool_flag = provider_options_map.at("so_epctx_embed_mode"); - if (bool_flag == "true" || bool_flag == "True") - so_epctx_embed_mode = true; - else if (bool_flag == "false" || bool_flag == "False") - so_epctx_embed_mode = false; - bool_flag = ""; - } - - if (provider_options_map.find("so_epctx_path") != provider_options_map.end()) { - // The path to dump epctx model is valid only when epctx is enabled. - // Overrides the cache_dir option to dump model cache files from OV. - if (export_ep_ctx_blob && - !provider_options_map.at("so_epctx_path").empty()) { - cache_dir = provider_options_map.at("so_epctx_path"); - auto file_path = std::filesystem::path(cache_dir); - // ep_context_file_path_ file extension must be .onnx - if (file_path.extension().generic_string() == ".onnx") { - // ep_context_file_path_ must be provided as a directory, create it if doesn't exist - auto parent_path = file_path.parent_path(); - if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && - !std::filesystem::create_directory(parent_path)) { - ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n"); - } - } else { - ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir + " \n"); - } - } - } return std::make_shared(const_cast(device_type.c_str()), const_cast(precision.c_str()), @@ -329,10 +298,8 @@ struct OpenVINO_Provider : Provider { context, enable_opencl_throttling, disable_dynamic_shapes, - export_ep_ctx_blob, - enable_qdq_optimizer, - disable_cpu_fallback, - so_epctx_embed_mode); + enable_qdq_optimizer, + config_options); } void Initialize() override { diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h b/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h index bff70a90b6a70..0cbf051c6df26 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h @@ -14,8 +14,7 @@ namespace onnxruntime { struct SessionOptions; // defined in provider_bridge_ort.cc struct OpenVINOProviderFactoryCreator { - static std::shared_ptr Create(ProviderOptions* provider_options_map, + static std::shared_ptr Create(const ProviderOptions* provider_options_map, const SessionOptions* session_options); - static std::shared_ptr Create(const OrtOpenVINOProviderOptions* provider_options); }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 4f062efb09e7f..ebd1e4b0e7105 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -578,6 +578,7 @@ struct ProviderHost { // ConfigOptions virtual std::optional ConfigOptions__GetConfigEntry(const ConfigOptions* p, const std::string& config_key) = 0; + virtual std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, const std::string& default_value) = 0; // OrtRunOptions virtual const ConfigOptions& RunOptions__GetConfigOptions(const RunOptions* p) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 63ef36b0a7942..2554997d2d13e 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -485,6 +485,10 @@ struct ConfigOptions final { return g_host->ConfigOptions__GetConfigEntry(this, config_key); } + std::string GetConfigOrDefault(const std::string& config_key, const std::string& default_value) const { + return g_host->ConfigOptions__GetConfigOrDefault(this, config_key, default_value); + } + PROVIDER_DISALLOW_ALL(ConfigOptions) }; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 8e807c375143e..20495abd71833 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -706,6 +706,11 @@ struct ProviderHostImpl : ProviderHost { return p->GetConfigEntry(config_key); } + // ConfigOptions (wrapped) + std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, const std::string& default_value) override { + return p->GetConfigOrDefault(config_key, default_value); + } + // OrtRunOptions (wrapped) const ConfigOptions& RunOptions__GetConfigOptions(const RunOptions* p) override { return p->config_options; } @@ -1809,49 +1814,18 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O // Add new provider option below ov_options_converted_map["num_streams"] = "1"; - ov_options_converted_map["export_ep_ctx_blob"] = "false"; + //ov_options_converted_map["export_ep_ctx_blob"] = "false"; ov_options_converted_map["model_priority"] = "DEFAULT"; ov_options_converted_map["enable_qdq_optimizer"] = "false"; return ov_options_converted_map; } -std::shared_ptr OpenVINOProviderFactoryCreator::Create(const OrtOpenVINOProviderOptions* provider_options) { - ProviderOptions ov_options_converted_map = onnxruntime::OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(provider_options); - return s_library_openvino.Get().CreateExecutionProviderFactory(&ov_options_converted_map); -} - -void ORTSessionOptionsToOrtOpenVINOProviderOptions(ProviderOptions& ov_options, - const SessionOptions* session_options) { - bool disable_cpu_fallback = session_options->config_options.GetConfigOrDefault( - kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; - if (disable_cpu_fallback) - ov_options["disable_cpu_fallback"] = "true"; - - // values from session options will override the providerOptions Value - bool so_epctx_enable = session_options->config_options.GetConfigOrDefault( - kOrtSessionOptionEpContextEnable, "0") == "1"; - if (so_epctx_enable) - ov_options["so_export_ep_ctx_blob"] = "true"; - - std::string so_cache_path = session_options->config_options.GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, "").c_str(); - ov_options["so_epctx_path"] = so_cache_path; - - // Default embedMode is 1. Saving the compiled model contents as a Epctx node attribute - bool so_epctx_embed_mode = session_options->config_options.GetConfigOrDefault( - kOrtSessionOptionEpContextEmbedMode, "1") == "0"; - if (so_epctx_embed_mode) { - // defaults to true - ov_options["so_epctx_embed_mode"] = "false"; - } -} - -std::shared_ptr OpenVINOProviderFactoryCreator::Create(ProviderOptions* provider_options_map, +std::shared_ptr OpenVINOProviderFactoryCreator::Create(const ProviderOptions* provider_options_map, const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. - if (session_options) { - onnxruntime::ORTSessionOptionsToOrtOpenVINOProviderOptions(*provider_options_map, session_options); - } - return s_library_openvino.Get().CreateExecutionProviderFactory(provider_options_map); + std::pair buffer = {provider_options_map, session_options->config_options}; + const void* obj = reinterpret_cast(&buffer); + return s_library_openvino.Get().CreateExecutionProviderFactory(obj); } std::shared_ptr DnnlProviderFactoryCreator::Create(const OrtDnnlProviderOptions* dnnl_options) { @@ -2106,7 +2080,8 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_MIGraphX, _In ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options) { API_IMPL_BEGIN - auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(provider_options); + const onnxruntime::ProviderOptions ov_options_converted_map = onnxruntime::OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(provider_options); + auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&ov_options_converted_map, &(options->value)); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "SessionOptionsAppendExecutionProvider_OpenVINO: Failed to load shared library"); } diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 6451f8ec6dce8..aecc4e4eee564 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -99,9 +99,10 @@ std::unique_ptr MIGraphXExecutionProviderWithOptions(const O return nullptr; } -std::unique_ptr OpenVINOExecutionProviderWithOptions(const OrtOpenVINOProviderOptions* params) { +std::unique_ptr OpenVINOExecutionProviderWithOptions(const ProviderOptions* params, + const SessionOptions* session_options) { #ifdef USE_OPENVINO - return OpenVINOProviderFactoryCreator::Create(params)->CreateProvider(); + return OpenVINOProviderFactoryCreator::Create(params, session_options)->CreateProvider(); #else ORT_UNUSED_PARAMETER(params); return nullptr; diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index b3a619022f79b..57da473454ccc 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -49,7 +49,7 @@ std::unique_ptr TensorrtExecutionProviderWithOptions(const O std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params); std::unique_ptr DefaultMIGraphXExecutionProvider(); std::unique_ptr MIGraphXExecutionProviderWithOptions(const OrtMIGraphXProviderOptions* params); -std::unique_ptr OpenVINOExecutionProviderWithOptions(const OrtOpenVINOProviderOptions* params); +std::unique_ptr OpenVINOExecutionProviderWithOptions(const ProviderOptions* params, const SessionOptions* session_options = nullptr); std::unique_ptr DefaultOpenVINOExecutionProvider(); std::unique_ptr DefaultNnapiExecutionProvider(); std::unique_ptr DefaultVSINPUExecutionProvider(); From 05802f451f1367b85ec7a28defb3ed5dd69d4337 Mon Sep 17 00:00:00 2001 From: sfatimar Date: Mon, 23 Sep 2024 23:31:13 +0530 Subject: [PATCH 10/32] Changes for adding config buffer --- .../openvino/openvino_provider_factory.cc | 16 ++++++++-------- .../shared_library/provider_interfaces.h | 3 ++- onnxruntime/core/session/provider_bridge_ort.cc | 14 ++++++++------ 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 7072ab3660c92..06e765c5ed2ff 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -50,7 +50,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { - bool so_disable_cpu_fallback = config_options_.GetConfigOrDefault("session.disable_cpu_ep_fallback", "0") == "1"; bool so_export_ep_ctx_blob = config_options_.GetConfigOrDefault("ep.context_enable", "0") == "1"; bool so_epctx_embed_mode = config_options_.GetConfigOrDefault("ep.context_embed_mode", "1") == "1"; @@ -65,7 +64,8 @@ std::unique_ptr OpenVINOProviderFactory::CreateProvider() { auto parent_path = file_path.parent_path(); if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && !std::filesystem::create_directory(parent_path)) { - ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n"); + ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + \ + file_path.parent_path().generic_string() + " \n"); } } else { ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir_ + " \n"); @@ -93,11 +93,11 @@ struct OpenVINO_Provider : Provider { void* GetInfo() override { return &g_info; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { - typedef std::pair buffer_t; - const buffer_t* buffer = reinterpret_cast(void_params); - - auto& provider_options_map = *buffer->first; - const ConfigOptions& config_options = buffer->second; + // Extract the void_params into ProviderOptions and ConfigOptions + typedef std::pair ConfigBuffer; + const ConfigBuffer* buffer = reinterpret_cast(void_params); + auto& provider_options_map = *buffer->first; + const ConfigOptions& config_options = buffer->second; std::string device_type = ""; // [device_type]: Overrides the accelerator hardware type and precision // with these values at runtime. @@ -298,7 +298,7 @@ struct OpenVINO_Provider : Provider { context, enable_opencl_throttling, disable_dynamic_shapes, - enable_qdq_optimizer, + enable_qdq_optimizer, config_options); } diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index ebd1e4b0e7105..9de2a7b69e9a4 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -578,7 +578,8 @@ struct ProviderHost { // ConfigOptions virtual std::optional ConfigOptions__GetConfigEntry(const ConfigOptions* p, const std::string& config_key) = 0; - virtual std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, const std::string& default_value) = 0; + virtual std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, \ + const std::string& default_value) = 0; // OrtRunOptions virtual const ConfigOptions& RunOptions__GetConfigOptions(const RunOptions* p) = 0; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 20495abd71833..113567dbf4f31 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -41,6 +41,7 @@ #include "core/session/onnxruntime_c_api.h" #include "core/common/string_helper.h" +#include #ifdef ENABLE_TRAINING #ifdef ENABLE_TRAINING_TORCH_INTEROP @@ -707,7 +708,8 @@ struct ProviderHostImpl : ProviderHost { } // ConfigOptions (wrapped) - std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, const std::string& default_value) override { + std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, \ + const std::string& default_value) override { return p->GetConfigOrDefault(config_key, default_value); } @@ -1814,17 +1816,16 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O // Add new provider option below ov_options_converted_map["num_streams"] = "1"; - //ov_options_converted_map["export_ep_ctx_blob"] = "false"; ov_options_converted_map["model_priority"] = "DEFAULT"; ov_options_converted_map["enable_qdq_optimizer"] = "false"; return ov_options_converted_map; } -std::shared_ptr OpenVINOProviderFactoryCreator::Create(const ProviderOptions* provider_options_map, +std::shared_ptr OpenVINOProviderFactoryCreator::Create(const ProviderOptions* provider_options_map, \ const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. - std::pair buffer = {provider_options_map, session_options->config_options}; - const void* obj = reinterpret_cast(&buffer); + std::pair config_buffer = {provider_options_map, session_options->config_options}; + const void* obj = reinterpret_cast(&config_buffer); return s_library_openvino.Get().CreateExecutionProviderFactory(obj); } @@ -2078,7 +2079,8 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_MIGraphX, _In API_IMPL_END } -ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options) { +ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, \ + _In_ const OrtOpenVINOProviderOptions* provider_options) { API_IMPL_BEGIN const onnxruntime::ProviderOptions ov_options_converted_map = onnxruntime::OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(provider_options); auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&ov_options_converted_map, &(options->value)); From feaa0fb8e8c22a74a23fee69ead19989fd6b8a2a Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Mon, 23 Sep 2024 07:50:43 -0700 Subject: [PATCH 11/32] fix psa psr accuracy issue --- .../providers/openvino/backends/basic_backend.cc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 1f9c61780f27a..ac203e111cece 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -306,9 +306,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); ov_tensor_data_t ov_tensor_key; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_key = it->second; - } else { + { // Does this make sense for both types of allocators? auto input = graph_input_info.at(input_idx); if (allocator_name == OpenVINO_RT_NPU) { @@ -319,7 +317,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_key.copy_needed = true; ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape()); } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); + ort_ov_tensor_map[ort_tensor_key] = ov_tensor_key; if (ov_tensor_key.copy_needed) { const char* ort_tensor_data = tensor.GetTensorData(); @@ -366,9 +364,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_data_t ov_tensor_data; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; - } else { + { auto output = graph_output_info.at(output_idx); if (allocator_name == OpenVINO_RT_NPU) { ov_tensor_data.copy_needed = false; @@ -378,7 +374,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_data.copy_needed = true; ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape()); } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); + ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; try { infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); From bbebca87bcd4f03225b3c5428698af3e2a7a5c63 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Sat, 14 Sep 2024 14:49:32 +0530 Subject: [PATCH 12/32] feat: Load custom json OV config during runtime --- cmake/onnxruntime_providers_openvino.cmake | 2 +- .../core/providers/openvino/backend_utils.cc | 23 +++++++ .../core/providers/openvino/backend_utils.h | 2 + .../openvino/backends/basic_backend.cc | 67 +++++++++++++++++++ .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_execution_provider.cc | 1 + .../openvino/openvino_execution_provider.h | 3 + .../openvino/openvino_provider_factory.cc | 15 +++-- onnxruntime/test/perftest/ort_test_session.cc | 4 +- 9 files changed, 112 insertions(+), 6 deletions(-) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 2eb3611bae902..5dcee285a5b13 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -37,7 +37,7 @@ source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_openvino_cc_srcs}) onnxruntime_add_shared_library_module(onnxruntime_providers_openvino ${onnxruntime_providers_openvino_cc_srcs} "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc") - onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx) + onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx nlohmann_json::nlohmann_json) install(FILES ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino/openvino_provider_factory.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/) set_target_properties(onnxruntime_providers_openvino PROPERTIES CXX_STANDARD 20) diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index f772b9c3b0478..75fa7edf0d7df 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -11,6 +11,7 @@ #include "core/providers/shared_library/provider_api.h" #include "core/providers/openvino/backend_utils.h" #include "core/providers/openvino/ov_interface.h" +#include "nlohmann/json.hpp" using Exception = ov::Exception; @@ -267,6 +268,28 @@ void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std printPerformanceCounts(performanceMap, stream, std::move(deviceName)); } +void LoadConfig(const std::string& filename, std::map& config) { + std::ifstream input_filestream(filename); + if (!input_filestream.is_open()) { + throw std::runtime_error("Can't load config file \"" + filename + "\"."); + } + + nlohmann::json json_config; + try { + input_filestream >> json_config; + } catch (const std::exception& e) { + throw std::runtime_error("Can't parse config file \"" + filename + "\".\n" + e.what()); + } + + for (auto item = json_config.cbegin(), end = json_config.cend(); item != end; ++item) { + const std::string& deviceName = item.key(); + const auto& item_value = item.value(); + for (auto option = item_value.cbegin(), item_value_end = item_value.cend(); option != item_value_end; ++option) { + config[deviceName][option.key()] = option.value().get(); + } + } +} + } // namespace backend_utils } // namespace openvino_ep } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h index 9e65770da7d23..a105e6b08aade 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.h +++ b/onnxruntime/core/providers/openvino/backend_utils.h @@ -70,6 +70,8 @@ void printPerformanceCounts(const std::vector& performanceMap, void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std::string deviceName); +void LoadConfig(const std::string& filename, std::map& config); + } // namespace backend_utils } // namespace openvino_ep } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index ac203e111cece..d9eb480fe96a7 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -178,6 +178,73 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } #endif } + +if (!global_context_.load_config.empty()) { + std::map target_config; + LoadConfig(global_context_.load_config, target_config); + + // Parse device types like "AUTO:CPU,GPU" and extract individual devices + auto parse_individual_devices = [&](const std::string& device_type) -> std::vector { + std::vector devices; + auto delimiter_pos = device_type.find(':'); + if (delimiter_pos != std::string::npos) { + std::stringstream str_stream(device_type.substr(delimiter_pos + 1)); + std::string device; + while (std::getline(str_stream, device, ',')) { + devices.emplace_back(device); + } + } else { + devices.emplace_back(device_type); + } + return devices; + }; + + // Check if a property is supported and mutable + auto is_supported_and_mutable = [&](const std::string& key, + const std::vector& supported_config) -> bool { + auto it = std::find_if(supported_config.begin(), supported_config.end(), [&](const ov::PropertyName& property) { + return property == key && property.is_mutable(); + }); + return it != supported_config.end(); + }; + + // Set properties if they are valid, else log a warning if the property is missing or immutable by skipping the same + auto set_target_properties = [&](const std::string& device, const ov::AnyMap& config_options, + const std::vector& supported_properties) { + for (const auto& [key, value] : config_options) { + if (is_supported_and_mutable(key, supported_properties)) { + global_context_.ie_core.Get().set_property(device, ov::AnyMap{{key, value}}); + } else { + LOGS_DEFAULT(INFO)<< "WARNING: Property \"" << key << "\" is either unsupported in current OpenVINO version\"" + << "\" or property is immutable for target device \"" << device << "\". Skipping setting this property."; + } + } + }; + + // Check if the device type is AUTO, HETERO, or MULTI + if (global_context_.device_type.find("AUTO") == 0 || + global_context_.device_type.find("HETERO") == 0 || + global_context_.device_type.find("MULTI") == 0) { + // Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"]) + auto individual_devices = parse_individual_devices(global_context_.device_type); + // Set properties only for individual devices (e.g., "CPU", "GPU") + for (const std::string& device : individual_devices) { + if (target_config.count(device)) { + // Get supported properties for each individual device + auto device_properties = global_context_.ie_core.Get().get_property(device, ov::supported_properties); + // Set properties for the device + set_target_properties(device, target_config.at(device), device_properties); + } + } + } else { + if (target_config.count(global_context_.device_type)) { + auto supported_properties = global_context_.ie_core.Get().get_property(global_context_.device_type, + ov::supported_properties); + set_target_properties(global_context_.device_type, + target_config.at(global_context_.device_type), supported_properties); + } + } +} } void BasicBackend::EnableCaching(ov::AnyMap& device_config) { diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 598e985676f8d..2d238917eb8ed 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -27,6 +27,7 @@ struct GlobalContext { std::string precision_str; std::string model_precision; std::string cache_dir; + std::string load_config; std::string model_priority = "DEFAULT"; int num_streams; std::vector deviceAvailableList = {true, true, true, true, true, true, true, true}; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 08144651319cf..c55e7a607e496 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -27,6 +27,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv global_context_->precision_str = info.precision_; global_context_->enable_npu_fast_compile = info.enable_npu_fast_compile_; global_context_->cache_dir = info.cache_dir_; + global_context_->load_config = info.load_config_; global_context_->model_priority = info.model_priority_; global_context_->num_streams = info.num_streams_; global_context_->context = info.context_; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 8b1c62c607f6e..a758ddaee6016 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -81,6 +81,7 @@ struct OpenVINOExecutionProviderInfo { std::string precision_{""}; bool enable_npu_fast_compile_{false}; size_t num_of_threads_{0}; + std::string load_config_{""}; std::string cache_dir_{""}; std::string model_priority_{""}; int num_streams_{1}; @@ -96,6 +97,7 @@ struct OpenVINOExecutionProviderInfo { explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision, bool enable_npu_fast_compile, size_t num_of_threads, + const std::string& load_config, const std::string& cache_dir, const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, @@ -104,6 +106,7 @@ struct OpenVINOExecutionProviderInfo { : precision_(std::move(precision)), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), + load_config_{load_config}, cache_dir_(std::move(cache_dir)), model_priority_(std::move(model_priority)), num_streams_(num_streams), diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 06e765c5ed2ff..4446c1f14b9e1 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -10,14 +10,14 @@ namespace onnxruntime { struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const char* device_type, const char* precision, bool enable_npu_fast_compile, size_t num_of_threads, - const char* cache_dir, const char* model_priority, + const std::string& load_config, const char* cache_dir, const char* model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, - bool enable_qdq_optimizer, const ConfigOptions& config_options - ) + bool enable_qdq_optimizer, const ConfigOptions& config_options) : precision_(precision), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), + load_config_(load_config), model_priority_(model_priority), num_streams_(num_streams), context_(context), @@ -39,6 +39,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { std::string precision_; bool enable_npu_fast_compile_; size_t num_of_threads_; + std::string load_config_; std::string cache_dir_; std::string model_priority_; int num_streams_; @@ -72,7 +73,7 @@ std::unique_ptr OpenVINOProviderFactory::CreateProvider() { } } - OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, + OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, load_config_, cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_, disable_dynamic_shapes_, so_export_ep_ctx_blob, enable_qdq_optimizer_, so_disable_cpu_fallback, so_epctx_embed_mode); @@ -110,6 +111,7 @@ struct OpenVINO_Provider : Provider { // speeds up the model's compilation to NPU device specific format. int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of // threads with this value at runtime. + std::string load_config = ""; // Path to JSON file to load custom OV parameters. std::string cache_dir = ""; // [cache_dir]: specify the path to // dump and load the blobs for the model caching/kernel caching (GPU) // feature. If blob files are already present, it will be directly loaded. @@ -200,6 +202,10 @@ struct OpenVINO_Provider : Provider { cache_dir = provider_options_map.at("cache_dir"); } + if (provider_options_map.find("load_config") != provider_options_map.end()) { + load_config = provider_options_map.at("load_config"); + } + if (provider_options_map.find("context") != provider_options_map.end()) { std::string str = provider_options_map.at("context"); uint64_t number = std::strtoull(str.c_str(), nullptr, 16); @@ -292,6 +298,7 @@ struct OpenVINO_Provider : Provider { const_cast(precision.c_str()), enable_npu_fast_compile, num_of_threads, + load_config, const_cast(cache_dir.c_str()), model_priority, num_streams, diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 403ba89a83a43..79463c1418aa4 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -831,6 +831,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else { ov_options[key] = value; } + } else if (key == "load_config") { + ov_options[key] = value; } else if (key == "model_priority") { ov_options[key] = value; } else if (key == "cache_dir") { @@ -855,7 +857,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else if (key == "device_memory_name") { device_memory_name_ = std::move(value); } else { - ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); + ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'load_config', 'cache_dir', 'num_streams', 'device_memory_name', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } session_options.AppendExecutionProvider_OpenVINO_V2(ov_options); From ab9de33fa3f951d9aec50459121cd067cfb7e042 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Wed, 25 Sep 2024 12:02:16 +0530 Subject: [PATCH 13/32] fix: review comment fixes --- .../core/providers/openvino/backend_utils.cc | 10 ++++--- .../openvino/backends/basic_backend.cc | 4 +-- .../openvino/openvino_execution_provider.h | 8 +++--- .../openvino/openvino_provider_factory.cc | 26 +++++++++---------- onnxruntime/test/perftest/ort_test_session.cc | 2 +- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index 75fa7edf0d7df..4d9fbe09f118d 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -271,14 +271,18 @@ void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std void LoadConfig(const std::string& filename, std::map& config) { std::ifstream input_filestream(filename); if (!input_filestream.is_open()) { - throw std::runtime_error("Can't load config file \"" + filename + "\"."); + ORT_THROW("Can't load config file \"" + filename + "\"."); } nlohmann::json json_config; try { input_filestream >> json_config; - } catch (const std::exception& e) { - throw std::runtime_error("Can't parse config file \"" + filename + "\".\n" + e.what()); + } catch (const OnnxRuntimeException& ex) { + ORT_THROW("Can't parse config file \"" + filename + "\".\n" + ex.what()); + } catch (const std::exception& ex) { + throw std::runtime_error("Standard exception for config file \"" + filename + "\".\n" + ex.what()); + } catch (...) { + throw std::runtime_error("Unknown exception for config file \"" + filename + "\".\n"); } for (auto item = json_config.cbegin(), end = json_config.cend(); item != end; ++item) { diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d9eb480fe96a7..e45ea29aa7ce2 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -215,8 +215,8 @@ if (!global_context_.load_config.empty()) { if (is_supported_and_mutable(key, supported_properties)) { global_context_.ie_core.Get().set_property(device, ov::AnyMap{{key, value}}); } else { - LOGS_DEFAULT(INFO)<< "WARNING: Property \"" << key << "\" is either unsupported in current OpenVINO version\"" - << "\" or property is immutable for target device \"" << device << "\". Skipping setting this property."; + LOGS_DEFAULT(WARNING)<< "WARNING: Property \"" << key << "\" is either unsupported in current OpenVINO version" + << " or property is immutable for target device \"" << device << "\". Skipping setting this property."; } } }; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index a758ddaee6016..cc09a4c6878b0 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -97,16 +97,16 @@ struct OpenVINOExecutionProviderInfo { explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision, bool enable_npu_fast_compile, size_t num_of_threads, - const std::string& load_config, - const std::string& cache_dir, const std::string& model_priority, - int num_streams, void* context, bool enable_opencl_throttling, + const std::string& load_config, const std::string& cache_dir, + const std::string& model_priority, int num_streams, + void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, bool enable_qdq_optimizer, bool disable_cpu_fallback, bool so_epctx_embed_mode) : precision_(std::move(precision)), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), - load_config_{load_config}, + load_config_(std::move(load_config)), cache_dir_(std::move(cache_dir)), model_priority_(std::move(model_priority)), num_streams_(num_streams), diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 4446c1f14b9e1..bc8b49da22eab 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -8,29 +8,27 @@ namespace onnxruntime { struct OpenVINOProviderFactory : IExecutionProviderFactory { - OpenVINOProviderFactory(const char* device_type, const char* precision, + OpenVINOProviderFactory(const std::string& device_type, const std::string& precision, bool enable_npu_fast_compile, size_t num_of_threads, - const std::string& load_config, const char* cache_dir, const char* model_priority, - int num_streams, void* context, + const std::string& load_config, const std::string& cache_dir, + const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool enable_qdq_optimizer, const ConfigOptions& config_options) - : precision_(precision), + : device_type_(device_type), + precision_(precision), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), load_config_(load_config), + cache_dir_(cache_dir), model_priority_(model_priority), num_streams_(num_streams), context_(context), enable_opencl_throttling_(enable_opencl_throttling), disable_dynamic_shapes_(disable_dynamic_shapes), enable_qdq_optimizer_(enable_qdq_optimizer), - config_options_(config_options) { - device_type_ = (device_type == nullptr) ? "" : device_type; - cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; -} + config_options_(config_options) {} - ~OpenVINOProviderFactory() override { - } + ~OpenVINOProviderFactory() override {} std::unique_ptr CreateProvider() override; @@ -115,7 +113,7 @@ struct OpenVINO_Provider : Provider { std::string cache_dir = ""; // [cache_dir]: specify the path to // dump and load the blobs for the model caching/kernel caching (GPU) // feature. If blob files are already present, it will be directly loaded. - const char* model_priority = "DEFAULT"; // High-level OpenVINO model priority hint + std::string model_priority = "DEFAULT"; // High-level OpenVINO model priority hint // Defines what model should be provided with more performant // bounded resource first int num_streams = 1; // [num_streams]: Option that specifies the number of parallel inference @@ -294,12 +292,12 @@ struct OpenVINO_Provider : Provider { } } - return std::make_shared(const_cast(device_type.c_str()), - const_cast(precision.c_str()), + return std::make_shared(device_type, + precision, enable_npu_fast_compile, num_of_threads, load_config, - const_cast(cache_dir.c_str()), + cache_dir, model_priority, num_streams, context, diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 79463c1418aa4..78f64c459d3b0 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -857,7 +857,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else if (key == "device_memory_name") { device_memory_name_ = std::move(value); } else { - ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'load_config', 'cache_dir', 'num_streams', 'device_memory_name', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); + ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'load_config', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } session_options.AppendExecutionProvider_OpenVINO_V2(ov_options); From 89b500630de533d7e1e338ea81126dacbd504b74 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Wed, 25 Sep 2024 14:52:15 +0530 Subject: [PATCH 14/32] fix: fix lint issues --- .../openvino/backends/basic_backend.cc | 124 +++++++++--------- .../openvino/openvino_provider_factory.cc | 28 ++-- .../shared_library/provider_interfaces.h | 2 +- .../core/session/provider_bridge_ort.cc | 13 +- onnxruntime/test/util/default_providers.cc | 2 +- 5 files changed, 86 insertions(+), 83 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index e45ea29aa7ce2..9001aa9db9a5a 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -179,73 +179,75 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { #endif } -if (!global_context_.load_config.empty()) { - std::map target_config; - LoadConfig(global_context_.load_config, target_config); - - // Parse device types like "AUTO:CPU,GPU" and extract individual devices - auto parse_individual_devices = [&](const std::string& device_type) -> std::vector { - std::vector devices; - auto delimiter_pos = device_type.find(':'); - if (delimiter_pos != std::string::npos) { - std::stringstream str_stream(device_type.substr(delimiter_pos + 1)); - std::string device; - while (std::getline(str_stream, device, ',')) { - devices.emplace_back(device); - } - } else { - devices.emplace_back(device_type); - } - return devices; - }; - - // Check if a property is supported and mutable - auto is_supported_and_mutable = [&](const std::string& key, - const std::vector& supported_config) -> bool { - auto it = std::find_if(supported_config.begin(), supported_config.end(), [&](const ov::PropertyName& property) { - return property == key && property.is_mutable(); - }); - return it != supported_config.end(); - }; - - // Set properties if they are valid, else log a warning if the property is missing or immutable by skipping the same - auto set_target_properties = [&](const std::string& device, const ov::AnyMap& config_options, - const std::vector& supported_properties) { - for (const auto& [key, value] : config_options) { - if (is_supported_and_mutable(key, supported_properties)) { - global_context_.ie_core.Get().set_property(device, ov::AnyMap{{key, value}}); + if (!global_context_.load_config.empty()) { + std::map target_config; + LoadConfig(global_context_.load_config, target_config); + + // Parse device types like "AUTO:CPU,GPU" and extract individual devices + auto parse_individual_devices = [&](const std::string& device_type) -> std::vector { + std::vector devices; + auto delimiter_pos = device_type.find(':'); + if (delimiter_pos != std::string::npos) { + std::stringstream str_stream(device_type.substr(delimiter_pos + 1)); + std::string device; + while (std::getline(str_stream, device, ',')) { + devices.emplace_back(device); + } } else { - LOGS_DEFAULT(WARNING)<< "WARNING: Property \"" << key << "\" is either unsupported in current OpenVINO version" - << " or property is immutable for target device \"" << device << "\". Skipping setting this property."; + devices.emplace_back(device_type); } - } - }; - - // Check if the device type is AUTO, HETERO, or MULTI - if (global_context_.device_type.find("AUTO") == 0 || - global_context_.device_type.find("HETERO") == 0 || - global_context_.device_type.find("MULTI") == 0) { - // Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"]) - auto individual_devices = parse_individual_devices(global_context_.device_type); - // Set properties only for individual devices (e.g., "CPU", "GPU") - for (const std::string& device : individual_devices) { - if (target_config.count(device)) { - // Get supported properties for each individual device - auto device_properties = global_context_.ie_core.Get().get_property(device, ov::supported_properties); - // Set properties for the device - set_target_properties(device, target_config.at(device), device_properties); + return devices; + }; + + // Check if a property is supported and mutable + auto is_supported_and_mutable = [&](const std::string& key, + const std::vector& supported_config) -> bool { + auto it = std::find_if(supported_config.begin(), supported_config.end(), [&](const ov::PropertyName& property) { + return property == key && property.is_mutable(); + }); + return it != supported_config.end(); + }; + + // Set properties if they are valid, else log a warning if the property is missing or immutable by skipping the same + auto set_target_properties = [&](const std::string& device, const ov::AnyMap& config_options, + const std::vector& supported_properties) { + for (const auto& [key, value] : config_options) { + if (is_supported_and_mutable(key, supported_properties)) { + global_context_.ie_core.Get().set_property(device, ov::AnyMap{{key, value}}); + } else { + LOGS_DEFAULT(WARNING) << "WARNING: Property \"" << key + << "\" is either unsupported in current OpenVINO version" + << " or property is immutable for target device \"" + << device << "\". Skipping setting this property."; + } + } + }; + + // Check if the device type is AUTO, HETERO, or MULTI + if (global_context_.device_type.find("AUTO") == 0 || + global_context_.device_type.find("HETERO") == 0 || + global_context_.device_type.find("MULTI") == 0) { + // Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"]) + auto individual_devices = parse_individual_devices(global_context_.device_type); + // Set properties only for individual devices (e.g., "CPU", "GPU") + for (const std::string& device : individual_devices) { + if (target_config.count(device)) { + // Get supported properties for each individual device + auto device_properties = global_context_.ie_core.Get().get_property(device, ov::supported_properties); + // Set properties for the device + set_target_properties(device, target_config.at(device), device_properties); + } + } + } else { + if (target_config.count(global_context_.device_type)) { + auto supported_properties = global_context_.ie_core.Get().get_property(global_context_.device_type, + ov::supported_properties); + set_target_properties(global_context_.device_type, + target_config.at(global_context_.device_type), supported_properties); } - } - } else { - if (target_config.count(global_context_.device_type)) { - auto supported_properties = global_context_.ie_core.Get().get_property(global_context_.device_type, - ov::supported_properties); - set_target_properties(global_context_.device_type, - target_config.at(global_context_.device_type), supported_properties); } } } -} void BasicBackend::EnableCaching(ov::AnyMap& device_config) { // cache_dir argument has no effect when working with an embed-mode EPContext Graph diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index bc8b49da22eab..c69d53638ae90 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -55,20 +55,20 @@ std::unique_ptr OpenVINOProviderFactory::CreateProvider() { std::string so_cache_path = config_options_.GetConfigOrDefault("ep.context_file_path", "").c_str(); if (so_export_ep_ctx_blob && !so_cache_path.empty()) { - cache_dir_ = so_cache_path; - auto file_path = std::filesystem::path(cache_dir_); - // ep_context_file_path_ file extension must be .onnx - if (file_path.extension().generic_string() == ".onnx") { - // ep_context_file_path_ must be provided as a directory, create it if doesn't exist - auto parent_path = file_path.parent_path(); - if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && - !std::filesystem::create_directory(parent_path)) { - ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + \ - file_path.parent_path().generic_string() + " \n"); - } - } else { - ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir_ + " \n"); - } + cache_dir_ = so_cache_path; + auto file_path = std::filesystem::path(cache_dir_); + // ep_context_file_path_ file extension must be .onnx + if (file_path.extension().generic_string() == ".onnx") { + // ep_context_file_path_ must be provided as a directory, create it if doesn't exist + auto parent_path = file_path.parent_path(); + if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && + !std::filesystem::create_directory(parent_path)) { + ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + + file_path.parent_path().generic_string() + " \n"); + } + } else { + ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir_ + " \n"); + } } OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, load_config_, diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 9de2a7b69e9a4..ad6a4d386deb1 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -578,7 +578,7 @@ struct ProviderHost { // ConfigOptions virtual std::optional ConfigOptions__GetConfigEntry(const ConfigOptions* p, const std::string& config_key) = 0; - virtual std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, \ + virtual std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, const std::string& default_value) = 0; // OrtRunOptions diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 113567dbf4f31..f0cf20a2b513f 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -708,7 +708,7 @@ struct ProviderHostImpl : ProviderHost { } // ConfigOptions (wrapped) - std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, \ + std::string ConfigOptions__GetConfigOrDefault(const ConfigOptions* p, const std::string& config_key, const std::string& default_value) override { return p->GetConfigOrDefault(config_key, default_value); } @@ -1821,10 +1821,11 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O return ov_options_converted_map; } -std::shared_ptr OpenVINOProviderFactoryCreator::Create(const ProviderOptions* provider_options_map, \ - const SessionOptions* session_options) { +std::shared_ptr OpenVINOProviderFactoryCreator::Create( + const ProviderOptions* provider_options_map, const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. - std::pair config_buffer = {provider_options_map, session_options->config_options}; + std::pair config_buffer = {provider_options_map, + session_options->config_options}; const void* obj = reinterpret_cast(&config_buffer); return s_library_openvino.Get().CreateExecutionProviderFactory(obj); } @@ -2079,8 +2080,8 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_MIGraphX, _In API_IMPL_END } -ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, \ - _In_ const OrtOpenVINOProviderOptions* provider_options) { +ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, + _In_ const OrtOpenVINOProviderOptions* provider_options) { API_IMPL_BEGIN const onnxruntime::ProviderOptions ov_options_converted_map = onnxruntime::OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(provider_options); auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&ov_options_converted_map, &(options->value)); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index aecc4e4eee564..20352d5cc8000 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -99,7 +99,7 @@ std::unique_ptr MIGraphXExecutionProviderWithOptions(const O return nullptr; } -std::unique_ptr OpenVINOExecutionProviderWithOptions(const ProviderOptions* params, +std::unique_ptr OpenVINOExecutionProviderWithOptions(const ProviderOptions* params, const SessionOptions* session_options) { #ifdef USE_OPENVINO return OpenVINOProviderFactoryCreator::Create(params, session_options)->CreateProvider(); From f9b995cee22da2fe703a84994140e1acae5e5c13 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Wed, 25 Sep 2024 05:05:05 -0700 Subject: [PATCH 15/32] Revert "fix psa psr accuracy issue" This reverts commit 60af87347a3c365072b61c02c333cb51a9d504d7. --- .../providers/openvino/backends/basic_backend.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 9001aa9db9a5a..59fcfb02c87e4 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -375,7 +375,9 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); ov_tensor_data_t ov_tensor_key; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - { + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_key = it->second; + } else { // Does this make sense for both types of allocators? auto input = graph_input_info.at(input_idx); if (allocator_name == OpenVINO_RT_NPU) { @@ -386,7 +388,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_key.copy_needed = true; ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape()); } - ort_ov_tensor_map[ort_tensor_key] = ov_tensor_key; + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); if (ov_tensor_key.copy_needed) { const char* ort_tensor_data = tensor.GetTensorData(); @@ -433,7 +435,9 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_data_t ov_tensor_data; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - { + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; + } else { auto output = graph_output_info.at(output_idx); if (allocator_name == OpenVINO_RT_NPU) { ov_tensor_data.copy_needed = false; @@ -443,7 +447,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_data.copy_needed = true; ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape()); } - ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); try { infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); From 717a9c39776b3d23da897c4c00867b6d360389ba Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Thu, 26 Sep 2024 12:02:56 +0530 Subject: [PATCH 16/32] update: Enable Python API for load_config in provider options --- onnxruntime/core/session/provider_bridge_ort.cc | 1 + onnxruntime/python/onnxruntime_pybind_state.cc | 3 +++ 2 files changed, 4 insertions(+) diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index f0cf20a2b513f..f10af05408717 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1816,6 +1816,7 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O // Add new provider option below ov_options_converted_map["num_streams"] = "1"; + ov_options_converted_map["load_config"] = ""; ov_options_converted_map["model_priority"] = "DEFAULT"; ov_options_converted_map["enable_qdq_optimizer"] = "false"; return ov_options_converted_map; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e8bf61612c89b..cbb3954b670ea 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1087,6 +1087,9 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.first == "num_streams") { OV_provider_options_map[option.first] = option.second; continue; + } else if (option.first == "load_config") { + OV_provider_options_map[option.first] = option.second; + continue; } else if (option.first == "cache_dir") { OV_provider_options_map[option.first] = option.second; continue; From 9999cd3509dc95920bcc1f9e5e8ed95dab16617d Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Mon, 30 Sep 2024 12:17:26 +0530 Subject: [PATCH 17/32] update: handling few edge cases for parsing --- onnxruntime/core/providers/openvino/backend_utils.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index 4d9fbe09f118d..7c040b50ff22c 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -285,6 +285,10 @@ void LoadConfig(const std::string& filename, std::map& throw std::runtime_error("Unknown exception for config file \"" + filename + "\".\n"); } + if (json_config.empty()) { + ORT_THROW("Empty JSON content passed \"" + filename + "\"."); + } + for (auto item = json_config.cbegin(), end = json_config.cend(); item != end; ++item) { const std::string& deviceName = item.key(); const auto& item_value = item.value(); From e95451bfc8a847b41cf15ae93f9b0da1b2bebef7 Mon Sep 17 00:00:00 2001 From: n1harika Date: Tue, 1 Oct 2024 11:22:39 +0530 Subject: [PATCH 18/32] added input checks for enable_qdq_optimiser (#460) --- .../core/providers/openvino/openvino_provider_factory.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index c69d53638ae90..cc19fc4debd98 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -268,6 +268,8 @@ struct OpenVINO_Provider : Provider { enable_qdq_optimizer = true; else if (bool_flag == "false" || bool_flag == "False") enable_qdq_optimizer = false; + else + ORT_THROW("[ERROR] [OpenVINO-EP] enable_qdq_optimiser should be a boolean.\n"); bool_flag = ""; } From 4db9fb02b14ee5f2c6a292601c070d85fa15ec37 Mon Sep 17 00:00:00 2001 From: n1harika Date: Tue, 1 Oct 2024 18:25:48 +0530 Subject: [PATCH 19/32] FP8 support on NPU (#462) * adding fp8 support on NPU * changed OV Version to 2024_3 --- onnxruntime/core/providers/openvino/ov_versions/data_ops.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index d9aa13ec1bba9..e1b36ba692307 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -281,6 +281,10 @@ void DataOps::populate_types_supported() { std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); supported_types_npu_.insert( std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); + supported_types_npu_.insert( + std::make_pair(V_2024_3, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT8E4M3FN)); + supported_types_npu_.insert( + std::make_pair(V_2024_3, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT8E4M3FNUZ)); supported_types_cpu_.insert( std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); From 710d30999fc817e102f246383fb174a06fc6b78f Mon Sep 17 00:00:00 2001 From: Ankit Maheshkar Date: Tue, 1 Oct 2024 18:36:07 +0530 Subject: [PATCH 20/32] fix: updated Expand Op properties (#463) --- onnxruntime/core/providers/openvino/ov_versions/data_ops.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index e1b36ba692307..a147875765800 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -332,6 +332,7 @@ void DataOps::populate_op_mode_supported() { no_dimension_supported_.push_back({"Equal", V_2022_1, {"CPU"}}); no_dimension_supported_.push_back({"Equal", V_2023_0, {"GPU"}}); no_dimension_supported_.push_back({"Expand", V_2023_3, {"CPU"}}); + no_dimension_supported_.push_back({"Expand", V_2024_3, {"CPU", "GPU"}}); no_dimension_supported_.push_back({"Floor", V_2020_4, {"All"}}); no_dimension_supported_.push_back({"Gather", V_2020_4, {"All"}}); no_dimension_supported_.push_back({"Identity", V_2023_0, {"All"}}); From f3fddd41fcad01a263ddb4a4a9717facfa28aaec Mon Sep 17 00:00:00 2001 From: n1harika Date: Tue, 1 Oct 2024 19:19:05 +0530 Subject: [PATCH 21/32] removing fasterRCNN and GPT2_LM_HEAD from openvino_disabled_tests[] (#461) --- onnxruntime/test/providers/cpu/model_tests.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 177647ab5be6b..3e04ebeaed3d7 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -570,7 +570,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("yolov3"), ORT_TSTR("LSTM_Seq_lens_unpacked"), ORT_TSTR("tinyyolov3"), - ORT_TSTR("faster_rcnn"), + //ORT_TSTR("faster_rcnn"), ORT_TSTR("mask_rcnn"), ORT_TSTR("coreml_FNS-Candy_ImageNet"), ORT_TSTR("tf_mobilenet_v2_1.0_224"), @@ -581,7 +581,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("candy"), ORT_TSTR("cntk_simple_seg"), - ORT_TSTR("GPT2_LM_HEAD"), + //ORT_TSTR("GPT2_LM_HEAD"), ORT_TSTR("mlperf_ssd_mobilenet_300"), ORT_TSTR("fp16_coreml_FNS-Candy"), ORT_TSTR("fp16_test_tiny_yolov2"), From 0dd876899450d4c07a0746dfa2af3ba97eeb1e45 Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 <110383850+jatinwadhwa921@users.noreply.github.com> Date: Tue, 1 Oct 2024 06:50:51 -0700 Subject: [PATCH 22/32] Refactor tensor initialization check for external weights and fixed lint issues (#464) --- .../openvino/backends/basic_backend.cc | 6 ++++-- .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_execution_provider.cc | 1 + .../openvino/ov_versions/capability.cc | 2 +- .../openvino/ov_versions/capability.h | 4 ++++ .../openvino/ov_versions/data_ops.cc | 21 ++++++++++++++----- .../providers/openvino/ov_versions/data_ops.h | 7 ++++--- 7 files changed, 31 insertions(+), 11 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index bfd79bb960dcd..ede988edea94b 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -83,7 +83,8 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr subgraph_context_.subgraph_name); ie_cnn_network_ = exe_network_.Get().get_runtime_model(); } else if (global_context_.export_ep_ctx_blob && - hw_target.find("NPU") != std::string::npos) { + hw_target.find("NPU") != std::string::npos && + !global_context_.has_external_weights) { std::shared_ptr ov_model; { const std::string model = model_proto->SerializeAsString(); @@ -93,7 +94,8 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr ov_model = global_context_.ie_core.Get().read_model(model, ov::Tensor()); } exe_network_ = OVExeNetwork(global_context_.ie_core.Get().compile_model(ov_model, hw_target, device_config)); - } else if ((!subgraph_context_.has_dynamic_input_shape) && + } else if (!global_context_.has_external_weights && + (!subgraph_context_.has_dynamic_input_shape) && ((hw_target.find("AUTO") == std::string::npos) || (global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) > 2))) { // Optimized OV compile_model API is supported with AUTO from version 2024.3 and above diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 2d238917eb8ed..115d92a3f2d8c 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -22,6 +22,7 @@ struct GlobalContext { bool export_ep_ctx_blob = false; bool enable_qdq_optimizer = false; bool disable_cpu_fallback = false; + bool has_external_weights = false; size_t num_of_threads; std::string device_type; std::string precision_str; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index c55e7a607e496..d2d4845fb0d27 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -125,6 +125,7 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, result = obj.Execute(); global_context_->is_wholly_supported_graph = obj.IsWhollySupportedGraph(); + global_context_->has_external_weights = obj.HasExternalWeights(); return result; } diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.cc b/onnxruntime/core/providers/openvino/ov_versions/capability.cc index 3fcaff4369c89..22326650a0e2a 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/capability.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/capability.cc @@ -59,7 +59,7 @@ std::vector> GetCapability::Execute() { // This is a list of initializers that nGraph considers as constants. Example weights, reshape shape etc. std::unordered_set ng_required_initializers; - const auto unsupported_nodes = data_ops_->GetUnsupportedNodeIndices(ng_required_initializers); + const auto unsupported_nodes = data_ops_->GetUnsupportedNodeIndices(ng_required_initializers, has_external_weights_); #ifndef NDEBUG if (openvino_ep::backend_utils::IsDebugEnabled()) { std::cout << "No of unsupported nodes " << unsupported_nodes.size() << std::endl; diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.h b/onnxruntime/core/providers/openvino/ov_versions/capability.h index 63c83158accf8..2f87c4c73d892 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/capability.h +++ b/onnxruntime/core/providers/openvino/ov_versions/capability.h @@ -16,6 +16,7 @@ class GetCapability { std::string device_type_; DataOps* data_ops_; bool is_wholly_supported_graph_ = false; + bool has_external_weights_ = false; public: GetCapability(const GraphViewer& graph_viewer_param, @@ -25,6 +26,9 @@ class GetCapability { bool IsWhollySupportedGraph() { return is_wholly_supported_graph_; } + bool HasExternalWeights() { + return has_external_weights_; + } }; } // namespace openvino_ep diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index a147875765800..081ed1f4b3f8d 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -588,11 +588,21 @@ bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { } } -bool DataOps::unsupported_op_mode(const Node* node) { +bool DataOps::unsupported_op_mode(const Node* node, bool& has_external_weights_) { bool result = false; const auto& optype = node->OpType(); const auto& initializers = graph_viewer_.GetAllInitializedTensors(); + for (const auto& tensor_pair : initializers) { + const ONNX_NAMESPACE::TensorProto* tensor_proto = tensor_pair.second; + // Check if the tensor exists and if it has an external data location + if (tensor_proto && tensor_proto->has_data_location() && + tensor_proto->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + has_external_weights_ = true; + break; + } + } + auto iter = op_list_.equal_range(optype); for (auto it = iter.first; it != iter.second; ++it) { auto ob = it->second; @@ -642,7 +652,7 @@ bool DataOps::dimension_unsupported(const Node* node) { return true; } -bool DataOps::node_is_supported(const NodeIndex node_idx) { +bool DataOps::node_is_supported(const NodeIndex node_idx, bool& has_external_weights_) { const auto& node = graph_viewer_.GetNode(node_idx); const auto& optype = node->OpType(); @@ -750,7 +760,7 @@ bool DataOps::node_is_supported(const NodeIndex node_idx) { } // Check 3a - if (domain == kOnnxDomain && unsupported_op_mode(node)) { + if (domain == kOnnxDomain && unsupported_op_mode(node, has_external_weights_)) { if (optype == "GatherElements") { return true; } @@ -765,11 +775,12 @@ bool DataOps::node_is_supported(const NodeIndex node_idx) { return true; } -std::vector DataOps::GetUnsupportedNodeIndices(std::unordered_set& ng_required_initializers) { +std::vector DataOps::GetUnsupportedNodeIndices(std::unordered_set& ng_required_initializers, + bool& has_external_weights_) { std::vector unsupported_nodes_idx; for (const auto& node_idx : graph_viewer_.GetNodesInTopologicalOrder()) { - if (node_is_supported(node_idx)) { + if (node_is_supported(node_idx, has_external_weights_)) { // Collect inputs that are initializers graph_viewer_.GetNode(node_idx)->ForEachDef([&ng_required_initializers, this](const NodeArg& node_arg, bool is_input) { diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h index 4c064b08405c1..31273bee9c863 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h @@ -70,9 +70,9 @@ class DataOps { void populate_types_supported(); bool op_is_supported(std::string name, std::vector& list); bool dimension_unsupported(const Node* node); - bool unsupported_op_mode(const Node* node); + bool unsupported_op_mode(const Node* node, bool& has_external_weights_); bool type_is_supported(const NodeArg* node_arg, bool is_initializer); - bool node_is_supported(const NodeIndex node_idx); + bool node_is_supported(const NodeIndex node_idx, bool& has_external_weights_); public: DataOps(const GraphViewer& graph_viewer_param, VersionNum ver, @@ -85,7 +85,8 @@ class DataOps { populate_types_supported(); } - virtual std::vector GetUnsupportedNodeIndices(std::unordered_set& ng_required_initializers); + virtual std::vector GetUnsupportedNodeIndices( + std::unordered_set& ng_required_initializers, bool& has_external_weights_); virtual bool IsOpSupportedOnlyInModel(std::string name); virtual bool SpecialConditionForClusterSizeOne( std::unordered_set& ng_required_initializers, const Node* node); From 90aa047ded9007820c9ab095f926f9757696042e Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Wed, 2 Oct 2024 09:49:54 -0700 Subject: [PATCH 23/32] Remove npu fast compile (#465) * Remove invalid provider option 'enable_npu_fast_compile' in OVEP * Fix lint issues --- .../core/session/onnxruntime_c_api.h | 2 -- .../core/providers/openvino/contexts.h | 1 - .../openvino/openvino_execution_provider.cc | 1 - .../openvino/openvino_execution_provider.h | 4 +-- .../openvino/openvino_provider_factory.cc | 25 +++++-------------- .../core/session/provider_bridge_ort.cc | 6 ----- .../python/onnxruntime_pybind_state.cc | 9 ------- .../test/perftest/command_args_parser.cc | 3 +-- onnxruntime/test/perftest/ort_test_session.cc | 21 +++------------- onnxruntime/test/providers/cpu/model_tests.cc | 4 +-- 10 files changed, 14 insertions(+), 62 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 39e0361b7ff4f..275e8b25fdb4e 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -632,7 +632,6 @@ typedef struct OrtMIGraphXProviderOptions { typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus OrtOpenVINOProviderOptions() : device_type{}, - enable_npu_fast_compile{}, device_id{}, num_of_threads{}, cache_dir{}, @@ -645,7 +644,6 @@ typedef struct OrtOpenVINOProviderOptions { * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" */ const char* device_type; - unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled const char* device_id; size_t num_of_threads; ///< 0 = Use default number of threads const char* cache_dir; // path is set to empty by default diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 115d92a3f2d8c..7762cd8014fd8 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -15,7 +15,6 @@ namespace openvino_ep { struct GlobalContext { OVCore ie_core; bool is_wholly_supported_graph = false; - bool enable_npu_fast_compile = false; bool enable_opencl_throttling = false; bool disable_dynamic_shapes = false; bool ep_context_embed_mode = true; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index d2d4845fb0d27..19a634818a442 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -25,7 +25,6 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv global_context_ = std::make_unique(); global_context_->device_type = info.device_type_; global_context_->precision_str = info.precision_; - global_context_->enable_npu_fast_compile = info.enable_npu_fast_compile_; global_context_->cache_dir = info.cache_dir_; global_context_->load_config = info.load_config_; global_context_->model_priority = info.model_priority_; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index cc09a4c6878b0..c79e4469b6ab3 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -79,7 +79,6 @@ static std::vector parseDevices(const std::string& device_string, struct OpenVINOExecutionProviderInfo { std::string device_type_{""}; std::string precision_{""}; - bool enable_npu_fast_compile_{false}; size_t num_of_threads_{0}; std::string load_config_{""}; std::string cache_dir_{""}; @@ -96,7 +95,7 @@ struct OpenVINOExecutionProviderInfo { OpenVINOExecutionProviderInfo() = delete; explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision, - bool enable_npu_fast_compile, size_t num_of_threads, + size_t num_of_threads, const std::string& load_config, const std::string& cache_dir, const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, @@ -104,7 +103,6 @@ struct OpenVINOExecutionProviderInfo { bool enable_qdq_optimizer, bool disable_cpu_fallback, bool so_epctx_embed_mode) : precision_(std::move(precision)), - enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), load_config_(std::move(load_config)), cache_dir_(std::move(cache_dir)), diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index cc19fc4debd98..905cb429c1d26 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -9,14 +9,13 @@ namespace onnxruntime { struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const std::string& device_type, const std::string& precision, - bool enable_npu_fast_compile, size_t num_of_threads, + size_t num_of_threads, const std::string& load_config, const std::string& cache_dir, const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool enable_qdq_optimizer, const ConfigOptions& config_options) : device_type_(device_type), precision_(precision), - enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), load_config_(load_config), cache_dir_(cache_dir), @@ -35,7 +34,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { private: std::string device_type_; std::string precision_; - bool enable_npu_fast_compile_; size_t num_of_threads_; std::string load_config_; std::string cache_dir_; @@ -71,7 +69,7 @@ std::unique_ptr OpenVINOProviderFactory::CreateProvider() { } } - OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, load_config_, + OpenVINOExecutionProviderInfo info(device_type_, precision_, num_of_threads_, load_config_, cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_, disable_dynamic_shapes_, so_export_ep_ctx_blob, enable_qdq_optimizer_, so_disable_cpu_fallback, so_epctx_embed_mode); @@ -105,8 +103,6 @@ struct OpenVINO_Provider : Provider { // Not setting precision will execute with optimized precision for // best inference latency. set Precision=ACCURACY for executing models // with input precision for best accuracy. - bool enable_npu_fast_compile = false; // [enable_npu_fast_compile]: Fast-compile may be optionally enabled to - // speeds up the model's compilation to NPU device specific format. int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of // threads with this value at runtime. std::string load_config = ""; // Path to JSON file to load custom OV parameters. @@ -123,10 +119,11 @@ struct OpenVINO_Provider : Provider { bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU // device (Reduces CPU Utilization when using GPU) - void* context = nullptr; + bool enable_qdq_optimizer = false; // Enables QDQ pruning for efficient inference latency with NPU - bool enable_qdq_optimizer = false; + void* context = nullptr; + std::string bool_flag = ""; if (provider_options_map.find("device_type") != provider_options_map.end()) { device_type = provider_options_map.at("device_type").c_str(); @@ -243,16 +240,6 @@ struct OpenVINO_Provider : Provider { << "Executing with num_streams=1"; } } - std::string bool_flag = ""; - if (provider_options_map.find("enable_npu_fast_compile") != provider_options_map.end()) { - bool_flag = provider_options_map.at("enable_npu_fast_compile"); - if (bool_flag == "true" || bool_flag == "True") - enable_npu_fast_compile = true; - else if (bool_flag == "false" || bool_flag == "False") - enable_npu_fast_compile = false; - bool_flag = ""; - } - if (provider_options_map.find("enable_opencl_throttling") != provider_options_map.end()) { bool_flag = provider_options_map.at("enable_opencl_throttling"); if (bool_flag == "true" || bool_flag == "True") @@ -292,11 +279,11 @@ struct OpenVINO_Provider : Provider { disable_dynamic_shapes = false; } } + bool_flag = ""; } return std::make_shared(device_type, precision, - enable_npu_fast_compile, num_of_threads, load_config, cache_dir, diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index f10af05408717..c444561efef54 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1788,12 +1788,6 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O if (legacy_ov_options->device_type != nullptr) ov_options_converted_map["device_type"] = legacy_ov_options->device_type; - if (legacy_ov_options->enable_npu_fast_compile) { - ov_options_converted_map["enable_npu_fast_compile"] = "false"; - } else { - ov_options_converted_map["enable_npu_fast_compile"] = "true"; - } - if (legacy_ov_options->num_of_threads != '\0') ov_options_converted_map["num_of_threads"] = std::to_string(legacy_ov_options->num_of_threads); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index b4de58323bab1..84c457e7a1f5d 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1062,12 +1062,6 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.first == "precision") { OV_provider_options_map[option.first] = option.second; continue; - } else if (option.first == "enable_npu_fast_compile") { - if (!(option.second == "True" || option.second == "true" || - option.second == "False" || option.second == "false")) { - ORT_THROW("Invalid value passed for enable_npu_fast_compile: ", option.second); - } - OV_provider_options_map[option.first] = option.second; } else if (option.first == "enable_opencl_throttling") { if (!(option.second == "True" || option.second == "true" || option.second == "False" || option.second == "false")) { @@ -1112,9 +1106,6 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.first == "context") { OV_provider_options_map[option.first] = option.second; continue; - } else if (option.first == "export_ep_ctx_blob") { - OV_provider_options_map[option.first] = option.second; - continue; } else if (option.first == "enable_qdq_optimizer") { OV_provider_options_map[option.first] = option.second; continue; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 6e811f4596eab..48bfe4b284d6c 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -76,11 +76,10 @@ namespace perftest { "\n" "\t [OpenVINO only] [device_type]: Overrides the accelerator hardware type and precision with these values at runtime.\n" "\t [OpenVINO only] [device_id]: Selects a particular hardware device for inference.\n" - "\t [OpenVINO only] [enable_npu_fast_compile]: Optionally enabled to speeds up the model's compilation on NPU device targets.\n" "\t [OpenVINO only] [num_of_threads]: Overrides the accelerator hardware type and precision with these values at runtime.\n" "\t [OpenVINO only] [cache_dir]: Explicitly specify the path to dump and load the blobs(Model caching) or cl_cache (Kernel Caching) files feature. If blob files are already present, it will be directly loaded.\n" "\t [OpenVINO only] [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU device(Reduces the CPU Utilization while using GPU) \n" - "\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU enable_npu_fast_compile|true num_of_threads|5 enable_opencl_throttling|true cache_dir|\"\"\"\n" + "\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU num_of_threads|5 enable_opencl_throttling|true cache_dir|\"\"\"\n" "\n" "\t [QNN only] [backend_path]: QNN backend path. e.g '/folderpath/libQnnHtp.so', '/folderpath/libQnnCpu.so'.\n" "\t [QNN only] [profiling_level]: QNN profiling level, options: 'basic', 'detailed', default 'off'.\n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 78f64c459d3b0..dccefad39616c 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -795,13 +795,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n"); } } - } else if (key == "enable_npu_fast_compile") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n"); - } } else if (key == "enable_opencl_throttling") { if (value == "true" || value == "True" || value == "false" || value == "False") { @@ -845,19 +838,13 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else { ov_options[key] = value; } - } else if (key == "export_ep_ctx_blob") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW( - "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " - "should be a boolean i.e. true or false. Default value is false.\n"); - } } else if (key == "device_memory_name") { device_memory_name_ = std::move(value); } else { - ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'load_config', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); + ORT_THROW( + "[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO." + " ['device_type', 'device_id', 'num_of_threads', 'load_config', 'cache_dir', 'num_streams', " + "'enable_opencl_throttling', 'disable_dynamic_shapes', 'enable_qdq_optimizer', 'model_priority'] \n"); } } session_options.AppendExecutionProvider_OpenVINO_V2(ov_options); diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 3e04ebeaed3d7..e3c86a137484f 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -570,7 +570,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("yolov3"), ORT_TSTR("LSTM_Seq_lens_unpacked"), ORT_TSTR("tinyyolov3"), - //ORT_TSTR("faster_rcnn"), + // ORT_TSTR("faster_rcnn"), ORT_TSTR("mask_rcnn"), ORT_TSTR("coreml_FNS-Candy_ImageNet"), ORT_TSTR("tf_mobilenet_v2_1.0_224"), @@ -581,7 +581,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("mlperf_ssd_resnet34_1200"), ORT_TSTR("candy"), ORT_TSTR("cntk_simple_seg"), - //ORT_TSTR("GPT2_LM_HEAD"), + // ORT_TSTR("GPT2_LM_HEAD"), ORT_TSTR("mlperf_ssd_mobilenet_300"), ORT_TSTR("fp16_coreml_FNS-Candy"), ORT_TSTR("fp16_test_tiny_yolov2"), From fc3b92e4d2becfe89b81f33a8d78ae57b3f59ff2 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Wed, 2 Oct 2024 14:17:30 +0530 Subject: [PATCH 24/32] fix: Securing load_config path parsing --- .../openvino/openvino_provider_factory.cc | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 905cb429c1d26..ba9212b8ce55c 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -199,6 +199,26 @@ struct OpenVINO_Provider : Provider { if (provider_options_map.find("load_config") != provider_options_map.end()) { load_config = provider_options_map.at("load_config"); + + // Enforce that the input path is absolute, reject if not + if (!std::filesystem::path(load_config).is_absolute()) { + throw std::invalid_argument("The config file path must be an absolute path: " + load_config); + } + + auto resolve_path = [&](const std::string& path) -> std::string { + std::filesystem::path fs_path = path; + // Canonicalize the path to resolve symbolic links and remove '..' or '.' + try { + fs_path = std::filesystem::canonical(fs_path); + } catch (const std::filesystem::filesystem_error& e) { + throw std::runtime_error("Error resolving config file path: " + std::string(e.what())); + } + return fs_path.string(); + }; + + // Expand and resolve the filename to its canonical form + std::string resolved_filename = resolve_path(load_config); + load_config = resolved_filename; } if (provider_options_map.find("context") != provider_options_map.end()) { From dd49e37f72bd2e549b918e4a67019e271f889ae6 Mon Sep 17 00:00:00 2001 From: Javier Martinez Date: Thu, 3 Oct 2024 05:55:00 -0700 Subject: [PATCH 25/32] Device memory refactor fix (#466) --- onnxruntime/test/perftest/ort_test_session.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index dccefad39616c..c78b634b071db 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -888,25 +888,30 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } auto transform_fcn = std::function(); + auto new_value = std::function&, Ort::ConstTensorTypeAndShapeInfo&)>(); if (device_memory_name_.empty()) { transform_fcn = [](int64_t input) { return input; }; + new_value = [](OrtAllocator*, const std::vector&, Ort::ConstTensorTypeAndShapeInfo&) { + return Ort::Value(nullptr); + }; } else { Ort::MemoryInfo memory_info = Ort::MemoryInfo(device_memory_name_.data(), OrtArenaAllocator, 0, OrtMemTypeCPUOutput); custom_allocator_ = std::make_unique(session_, memory_info); allocator_ = *custom_allocator_; + + // free dimensions are treated as 1 if not overridden transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + new_value = [](OrtAllocator* allocator, const std::vector& output_shape, Ort::ConstTensorTypeAndShapeInfo& tensor_info) { + return Ort::Value::CreateTensor(allocator, output_shape.data(), output_shape.size(), tensor_info.GetElementType()); + }; } for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - - // free dimensions are treated as 1 if not overridden std::vector output_shape = tensor_info.GetShape(); std::transform(output_shape.begin(), output_shape.end(), output_shape.begin(), transform_fcn); - - outputs_.push_back(Ort::Value::CreateTensor(allocator_, output_shape.data(), - output_shape.size(), tensor_info.GetElementType())); + outputs_.emplace_back(new_value(allocator_, output_shape, tensor_info)); } } From d7ac0acece4c34551f4ac6387dfcc08a2250c813 Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 <110383850+jatinwadhwa921@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:48:55 +0530 Subject: [PATCH 26/32] Fixed coverity issues (#471) --- .../core/providers/openvino/backends/basic_backend.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index ede988edea94b..be46733a39190 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -346,7 +346,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque input_tensor_shape[tensor_iter] = *i; tensor_iter += 1; } - auto input = graph_input_info.at(input_idx); + const auto& input = graph_input_info.at(input_idx); OVTensorPtr tensor_ptr; // avoid input copies on the CPU device if (global_context_.device_type.find("CPU") != std::string::npos) { @@ -387,7 +387,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; try { - infer_request->SetTensor(input_name, ov_tensor_data.tensor_ptr); + infer_request->SetTensor(std::move(input_name), ov_tensor_data.tensor_ptr); } catch (const char* msg) { ORT_THROW(msg); } @@ -425,14 +425,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque if ((it == ort_ov_tensor_map.end()) || (it != ort_ov_tensor_map.end() && (it->second.ort_ptr != tensor.GetTensorRawData()))) { ov_tensor_data_t ov_tensor_data; - auto output = graph_output_info.at(output_idx); + const auto& output = graph_output_info.at(output_idx); ov_tensor_data.ort_ptr = tensor.GetTensorRawData(); ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), const_cast(tensor.GetTensorRawData())); ort_ov_tensor_map[ort_tensor_key] = ov_tensor_data; try { - infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); + infer_request->SetTensor(std::move(output_name), ov_tensor_data.tensor_ptr); } catch (const char* msg) { ORT_THROW(msg); } From a22166f960e022fa42199e5da733a515d1f19133 Mon Sep 17 00:00:00 2001 From: Ankit Maheshkar Date: Sun, 13 Oct 2024 23:54:59 +0530 Subject: [PATCH 27/32] update: Using load_config as map (#470) --- .../core/providers/openvino/backend_utils.cc | 31 ----- .../core/providers/openvino/backend_utils.h | 2 - .../openvino/backends/basic_backend.cc | 3 +- .../core/providers/openvino/contexts.h | 3 +- .../openvino/openvino_execution_provider.h | 5 +- .../openvino/openvino_provider_factory.cc | 121 ++++++++++++------ onnxruntime/test/perftest/ort_test_session.cc | 26 +++- 7 files changed, 113 insertions(+), 78 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index 7c040b50ff22c..f772b9c3b0478 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -11,7 +11,6 @@ #include "core/providers/shared_library/provider_api.h" #include "core/providers/openvino/backend_utils.h" #include "core/providers/openvino/ov_interface.h" -#include "nlohmann/json.hpp" using Exception = ov::Exception; @@ -268,36 +267,6 @@ void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std printPerformanceCounts(performanceMap, stream, std::move(deviceName)); } -void LoadConfig(const std::string& filename, std::map& config) { - std::ifstream input_filestream(filename); - if (!input_filestream.is_open()) { - ORT_THROW("Can't load config file \"" + filename + "\"."); - } - - nlohmann::json json_config; - try { - input_filestream >> json_config; - } catch (const OnnxRuntimeException& ex) { - ORT_THROW("Can't parse config file \"" + filename + "\".\n" + ex.what()); - } catch (const std::exception& ex) { - throw std::runtime_error("Standard exception for config file \"" + filename + "\".\n" + ex.what()); - } catch (...) { - throw std::runtime_error("Unknown exception for config file \"" + filename + "\".\n"); - } - - if (json_config.empty()) { - ORT_THROW("Empty JSON content passed \"" + filename + "\"."); - } - - for (auto item = json_config.cbegin(), end = json_config.cend(); item != end; ++item) { - const std::string& deviceName = item.key(); - const auto& item_value = item.value(); - for (auto option = item_value.cbegin(), item_value_end = item_value.cend(); option != item_value_end; ++option) { - config[deviceName][option.key()] = option.value().get(); - } - } -} - } // namespace backend_utils } // namespace openvino_ep } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h index a105e6b08aade..9e65770da7d23 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.h +++ b/onnxruntime/core/providers/openvino/backend_utils.h @@ -70,8 +70,6 @@ void printPerformanceCounts(const std::vector& performanceMap, void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std::string deviceName); -void LoadConfig(const std::string& filename, std::map& config); - } // namespace backend_utils } // namespace openvino_ep } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index be46733a39190..8a1844544328c 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -182,8 +182,7 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } if (!global_context_.load_config.empty()) { - std::map target_config; - LoadConfig(global_context_.load_config, target_config); + const std::map& target_config = global_context_.load_config; // Parse device types like "AUTO:CPU,GPU" and extract individual devices auto parse_individual_devices = [&](const std::string& device_type) -> std::vector { diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 7762cd8014fd8..a2f4b236213cc 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include "core/providers/openvino/ov_interface.h" @@ -27,7 +28,7 @@ struct GlobalContext { std::string precision_str; std::string model_precision; std::string cache_dir; - std::string load_config; + std::map load_config; std::string model_priority = "DEFAULT"; int num_streams; std::vector deviceAvailableList = {true, true, true, true, true, true, true, true}; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index c79e4469b6ab3..27a358507be5d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -80,7 +80,7 @@ struct OpenVINOExecutionProviderInfo { std::string device_type_{""}; std::string precision_{""}; size_t num_of_threads_{0}; - std::string load_config_{""}; + std::map load_config_{}; std::string cache_dir_{""}; std::string model_priority_{""}; int num_streams_{1}; @@ -96,7 +96,8 @@ struct OpenVINOExecutionProviderInfo { explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision, size_t num_of_threads, - const std::string& load_config, const std::string& cache_dir, + const std::map& load_config, + const std::string& cache_dir, const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index ba9212b8ce55c..b46106db3c232 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -1,16 +1,19 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License +#include +#include #include "core/providers/shared_library/provider_api.h" #include "core/providers/openvino/openvino_provider_factory.h" #include "core/providers/openvino/openvino_execution_provider.h" #include "core/providers/openvino/openvino_provider_factory_creator.h" +#include "nlohmann/json.hpp" namespace onnxruntime { struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const std::string& device_type, const std::string& precision, size_t num_of_threads, - const std::string& load_config, const std::string& cache_dir, + const std::map& load_config, const std::string& cache_dir, const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool enable_qdq_optimizer, const ConfigOptions& config_options) @@ -35,7 +38,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { std::string device_type_; std::string precision_; size_t num_of_threads_; - std::string load_config_; + const std::map load_config_; std::string cache_dir_; std::string model_priority_; int num_streams_; @@ -96,28 +99,30 @@ struct OpenVINO_Provider : Provider { auto& provider_options_map = *buffer->first; const ConfigOptions& config_options = buffer->second; - std::string device_type = ""; // [device_type]: Overrides the accelerator hardware type and precision - // with these values at runtime. - std::string precision = ""; // [precision]: Sets the inference precision for execution. - // Supported precision for devices are CPU=FP32, GPU=FP32,FP16, NPU=FP16. - // Not setting precision will execute with optimized precision for - // best inference latency. set Precision=ACCURACY for executing models - // with input precision for best accuracy. - int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of - // threads with this value at runtime. - std::string load_config = ""; // Path to JSON file to load custom OV parameters. - std::string cache_dir = ""; // [cache_dir]: specify the path to - // dump and load the blobs for the model caching/kernel caching (GPU) - // feature. If blob files are already present, it will be directly loaded. - std::string model_priority = "DEFAULT"; // High-level OpenVINO model priority hint - // Defines what model should be provided with more performant - // bounded resource first - int num_streams = 1; // [num_streams]: Option that specifies the number of parallel inference - // requests to be processed on a given `device_type`. Overrides the - // accelerator default value of number of streams - // with this value at runtime. - bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU - // device (Reduces CPU Utilization when using GPU) + std::string device_type = ""; // [device_type]: Overrides the accelerator hardware type and + // precision with these values at runtime. + std::string precision = ""; // [precision]: Sets the inference precision for execution. + // Supported precision for devices are + // CPU=FP32, GPU=FP32,FP16, NPU=FP16. + // Not setting precision will execute with optimized precision for + // best inference latency. set Precision=ACCURACY for executing + // models with input precision for best accuracy. + int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of + // number of threads with this value at runtime. + std::map load_config; // JSON config map to load custom OV parameters. + std::string cache_dir = ""; // [cache_dir]: specify the path to + // dump and load the blobs for the model caching/kernel caching + // (GPU) feature. If blob files are already present, + // it will be directly loaded. + std::string model_priority = "DEFAULT"; // High-level OpenVINO model priority hint + // Defines what model should be provided with more performant + // bounded resource first + int num_streams = 1; // [num_streams]: Option that specifies the number of parallel + // inference requests to be processed on a given `device_type`. + // Overrides the accelerator default value of number of streams + // with this value at runtime. + bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for + // GPU device (Reduces CPU Utilization when using GPU) bool enable_qdq_optimizer = false; // Enables QDQ pruning for efficient inference latency with NPU @@ -198,27 +203,65 @@ struct OpenVINO_Provider : Provider { } if (provider_options_map.find("load_config") != provider_options_map.end()) { - load_config = provider_options_map.at("load_config"); + auto parse_config = [&](const std::string& config_str) -> std::map { + // If the config string is empty, return an empty map and skip processing + if (config_str.empty()) { + LOGS_DEFAULT(WARNING) << "Empty OV Config Map passed. Skipping load_config option parsing.\n"; + return {}; + } - // Enforce that the input path is absolute, reject if not - if (!std::filesystem::path(load_config).is_absolute()) { - throw std::invalid_argument("The config file path must be an absolute path: " + load_config); - } + std::stringstream input_str_stream(config_str); + std::map target_map; - auto resolve_path = [&](const std::string& path) -> std::string { - std::filesystem::path fs_path = path; - // Canonicalize the path to resolve symbolic links and remove '..' or '.' try { - fs_path = std::filesystem::canonical(fs_path); - } catch (const std::filesystem::filesystem_error& e) { - throw std::runtime_error("Error resolving config file path: " + std::string(e.what())); + nlohmann::json json_config = nlohmann::json::parse(input_str_stream); + + if (!json_config.is_object()) { + ORT_THROW("Invalid JSON structure: Expected an object at the root."); + } + + for (auto& [key, value] : json_config.items()) { + ov::AnyMap inner_map; + + // Ensure the key is one of "CPU", "GPU", or "NPU" + if (key != "CPU" && key != "GPU" && key != "NPU") { + LOGS_DEFAULT(WARNING) << "Unsupported device key: " << key << ". Skipping entry.\n"; + continue; + } + + // Ensure that the value for each device is an object (PROPERTY -> VALUE) + if (!value.is_object()) { + ORT_THROW("Invalid JSON structure: Expected an object for device properties."); + } + + for (auto& [inner_key, inner_value] : value.items()) { + if (inner_value.is_string()) { + inner_map[inner_key] = inner_value.get(); + } else if (inner_value.is_number_integer()) { + inner_map[inner_key] = inner_value.get(); + } else if (inner_value.is_number_float()) { + inner_map[inner_key] = inner_value.get(); + } else if (inner_value.is_boolean()) { + inner_map[inner_key] = inner_value.get(); + } else { + LOGS_DEFAULT(WARNING) << "Unsupported JSON value type for key: " << inner_key << ". Skipping key."; + } + } + target_map[key] = inner_map; + } + } catch (const nlohmann::json::parse_error& e) { + // Handle syntax errors in JSON + ORT_THROW("JSON parsing error: " + std::string(e.what())); + } catch (const nlohmann::json::type_error& e) { + // Handle invalid type accesses + ORT_THROW("JSON type error: " + std::string(e.what())); + } catch (const std::exception& e) { + ORT_THROW("Error parsing load_config Map: " + std::string(e.what())); } - return fs_path.string(); + return target_map; }; - // Expand and resolve the filename to its canonical form - std::string resolved_filename = resolve_path(load_config); - load_config = resolved_filename; + load_config = parse_config(provider_options_map.at("load_config")); } if (provider_options_map.find("context") != provider_options_map.end()) { diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index c78b634b071db..3c5a0f8ff4faa 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -18,6 +18,10 @@ #include "providers.h" #include "TestCase.h" +#ifdef USE_OPENVINO +#include "nlohmann/json.hpp" +#endif + #ifdef USE_DML #include "core/providers/dml/dml_provider_factory.h" #include "core/providers/dml/dml_session_options_config_keys.h" @@ -825,7 +829,27 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); ov_options[key] = value; } } else if (key == "load_config") { - ov_options[key] = value; + auto load_json = [&](std::string filename) -> std::string { + std::ifstream input_filestream(filename); + if (!input_filestream.is_open()) { + ORT_THROW("Passed an invalid JSON config file path \"" + filename + "\"."); + } + nlohmann::json json_config; + try { + input_filestream >> json_config; + } catch (const OnnxRuntimeException& ex) { + ORT_THROW("Exception parsing config file \"" + filename + "\".\n" + ex.what()); + } catch (const std::exception& ex) { + throw std::runtime_error("Standard exception for config file \"" + filename + "\".\n" + ex.what()); + } catch (...) { + throw std::runtime_error("Unknown exception for config file \"" + filename + "\".\n"); + } + if (json_config.empty()) { + ORT_THROW("Empty JSON content passed \"" + filename + "\"."); + } + return json_config.dump(); + }; + ov_options[key] = load_json(value); } else if (key == "model_priority") { ov_options[key] = value; } else if (key == "cache_dir") { From 6089543d31a6d8b864b47f67497608f419518e9f Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 <110383850+jatinwadhwa921@users.noreply.github.com> Date: Mon, 14 Oct 2024 00:08:34 +0530 Subject: [PATCH 28/32] Upgrade OpenVINO Toolkit v2024.4 (#472) --- .../core/providers/openvino/ov_versions/capability.cc | 8 ++++---- .../core/providers/openvino/ov_versions/data_ops.cc | 8 ++++---- .../core/providers/openvino/ov_versions/data_ops.h | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.cc b/onnxruntime/core/providers/openvino/ov_versions/capability.cc index 22326650a0e2a..0d7ac64d86e68 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/capability.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/capability.cc @@ -35,16 +35,16 @@ GetCapability::GetCapability(const GraphViewer& graph_viewer_param, device_type_ = "CPU"; if (enable_qdq_optimizer) npu_qdq_optimizer_enabled = true; } -#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 0 - data_ops_ = new DataOps(graph_viewer_, V_2024_0, device_type_, npu_qdq_optimizer_enabled); -#elif OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 1 +#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 1 data_ops_ = new DataOps(graph_viewer_, V_2024_1, device_type_, npu_qdq_optimizer_enabled); #elif OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 2 data_ops_ = new DataOps(graph_viewer_, V_2024_2, device_type_, npu_qdq_optimizer_enabled); #elif OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 3 data_ops_ = new DataOps(graph_viewer_, V_2024_3, device_type_, npu_qdq_optimizer_enabled); +#elif OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 + data_ops_ = new DataOps(graph_viewer_, V_2024_4, device_type_, npu_qdq_optimizer_enabled); #else - data_ops_ = new DataOps(graph_viewer_, V_2024_3, device_type_, npu_qdq_optimizer_enabled); + data_ops_ = new DataOps(graph_viewer_, V_2024_4, device_type_, npu_qdq_optimizer_enabled); #endif } diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index 081ed1f4b3f8d..e8f6ae0a43734 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -368,7 +368,7 @@ void DataOps::populate_op_mode_supported() { // populate unsupportedmode_t { - UnsupportedOpMode obj = {{V_2024_1, V_2024_2, V_2024_3}, + UnsupportedOpMode obj = {{V_2024_1, V_2024_2, V_2024_3, V_2024_4}, [this](const Node* node, const InitializedTensorSet&) { // If the Input of ReduceMax op is UINT8, it is rejected (Due to output mismatch) for (size_t i = 0; i < node->InputDefs().size(); i++) { @@ -383,7 +383,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"ReduceMax", obj}); } { - UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3}, + UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3, V_2024_4}, [this](const Node* node, const InitializedTensorSet&) { const auto& input_arg = node->InputDefs()[1]; auto shape = input_arg->Shape(); @@ -400,7 +400,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Reshape", obj}); } { - UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3}, + UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3, V_2024_4}, [this](const Node* node, const InitializedTensorSet&) { // If the operator is unsqueeze // If axes is an input, then we cannot produce a static graph. @@ -415,7 +415,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Unsqueeze", obj}); } { - UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3}, + UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3, V_2024_4}, [this](const Node* node, const InitializedTensorSet&) { // check for attributes auto& upsample_attr = node->GetAttributes(); diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h index 31273bee9c863..5cd4c8658fb77 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h @@ -30,7 +30,8 @@ enum versionNum { V_2024_0, V_2024_1, V_2024_2, - V_2024_3 + V_2024_3, + V_2024_4 }; using VersionNum = enum versionNum; From 463328e449ded50ba6598d3f275f1f6a69d4f2e1 Mon Sep 17 00:00:00 2001 From: sfatimar Date: Mon, 14 Oct 2024 11:24:39 +0530 Subject: [PATCH 29/32] Commit Unused parameter for session_options --- onnxruntime/test/util/default_providers.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 20352d5cc8000..f05f13fbfc574 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -105,6 +105,7 @@ std::unique_ptr OpenVINOExecutionProviderWithOptions(const P return OpenVINOProviderFactoryCreator::Create(params, session_options)->CreateProvider(); #else ORT_UNUSED_PARAMETER(params); + ORT_UNUSED_PARAMETER(session_options); return nullptr; #endif } From a8350bbaa016b8f82f7621c2fe6b1a1bf966ed4e Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Mon, 14 Oct 2024 14:44:31 +0530 Subject: [PATCH 30/32] Retain the depreacted provider option with a warning --- cmake/CMakeLists.txt | 1 + include/onnxruntime/core/session/onnxruntime_c_api.h | 2 ++ .../core/providers/openvino/openvino_execution_provider.h | 2 +- onnxruntime/core/session/provider_bridge_ort.cc | 3 +++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 2e9be26fb9920..ef208f59f63b0 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1352,6 +1352,7 @@ if (onnxruntime_USE_OPENVINO) add_definitions(-DUSE_OPENVINO=1) if(onnxruntime_NPU_NO_FALLBACK) + add_definitions(-DOPENVINO_CONFIG_NPU=1) add_definitions(-DOPENVINO_DISABLE_NPU_FALLBACK=1) endif() diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 9d376d6dcf24b..9e71997c1e442 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -632,6 +632,7 @@ typedef struct OrtMIGraphXProviderOptions { typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus OrtOpenVINOProviderOptions() : device_type{}, + enable_npu_fast_compile{}, device_id{}, num_of_threads{}, cache_dir{}, @@ -644,6 +645,7 @@ typedef struct OrtOpenVINOProviderOptions { * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" */ const char* device_type; + unsigned char enable_npu_fast_compile; const char* device_id; size_t num_of_threads; ///< 0 = Use default number of threads const char* cache_dir; // path is set to empty by default diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 27a358507be5d..7d9da65ea7e07 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -94,7 +94,7 @@ struct OpenVINOExecutionProviderInfo { OpenVINOExecutionProviderInfo() = delete; - explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision, + explicit OpenVINOExecutionProviderInfo(std::string dev_type, const std::string& precision, size_t num_of_threads, const std::map& load_config, const std::string& cache_dir, diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 09b7863962496..2c4bffa4fb79f 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1810,6 +1810,9 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O ov_options_converted_map["disable_dynamic_shapes"] = "true"; } + if (legacy_ov_options->enable_npu_fast_compile) { + LOGS_DEFAULT(WARNING) << "enable_npu_fast_compile option is deprecated. Skipping this option"; + } // Add new provider option below ov_options_converted_map["num_streams"] = "1"; ov_options_converted_map["load_config"] = ""; From 14bdb3a753ba8a41507a47827ff7478b8042a734 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Mon, 28 Oct 2024 21:37:40 -0700 Subject: [PATCH 31/32] Tejal api doc changes (#469) * Updated API Documentation with new attributes * Revert "Merge branch 'microsoft:main' into tejal_api_doc_changes" This reverts commit 35df7d85901d345288d7b0324f963b4f28290d15, reversing changes made to faae40be85339bb72353e3d2a194456b51951404. * Revert "Revert "Merge branch 'microsoft:main' into tejal_api_doc_changes"" This reverts commit fdde6cc1a1275e7ffc67a1626d8ce72b6216f27c. * Revert "Revert "Revert "Merge branch 'microsoft:main' into tejal_api_doc_changes""" This reverts commit 162b69791b75189797d2cfa1c0f2bb77cecf035a. * Revert "Revert "Revert "Revert "Merge branch 'microsoft:main' into tejal_api_doc_changes"""" This reverts commit ec986f1543848b0ce08ef466001a0e53ee453a01. * Modified provider options with necessary changes and updated comments * changed comments --------- Co-authored-by: TejalKhade28 Co-authored-by: Vishnudas Thaniel S Co-authored-by: jatinwadhwa921 --- .../onnxruntime/core/session/onnxruntime_c_api.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 9e71997c1e442..2f10b9a2004a6 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -626,8 +626,13 @@ typedef struct OrtMIGraphXProviderOptions { } OrtMIGraphXProviderOptions; /** \brief OpenVINO Provider Options - * - * \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO + * \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility. + * \brief For latest OpenVINO Provider Options update to the ProviderOptions map. + * \brief Latest OpenVINO Provider Options are listed in the + * \htmlonly + * onnxruntime document. + * \endhtmlonly + * \see OrtApi::SessionOptionsAppendExecutionProvider() */ typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus @@ -647,9 +652,9 @@ typedef struct OrtOpenVINOProviderOptions { const char* device_type; unsigned char enable_npu_fast_compile; const char* device_id; - size_t num_of_threads; ///< 0 = Use default number of threads - const char* cache_dir; // path is set to empty by default - void* context; + size_t num_of_threads; ///< 0 = Use default number of threads + const char* cache_dir; ///< Any valid string path on the hardware target + void* context; ///< OpenCL Context unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled } OrtOpenVINOProviderOptions; From 4c7668bf6291e69df7b1eb953b695b22760a2917 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Tue, 29 Oct 2024 10:08:08 +0530 Subject: [PATCH 32/32] Revert "Tejal api doc changes (#469)" This reverts commit 14bdb3a753ba8a41507a47827ff7478b8042a734. --- .../onnxruntime/core/session/onnxruntime_c_api.h | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 2f10b9a2004a6..9e71997c1e442 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -626,13 +626,8 @@ typedef struct OrtMIGraphXProviderOptions { } OrtMIGraphXProviderOptions; /** \brief OpenVINO Provider Options - * \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility. - * \brief For latest OpenVINO Provider Options update to the ProviderOptions map. - * \brief Latest OpenVINO Provider Options are listed in the - * \htmlonly - * onnxruntime document. - * \endhtmlonly - * \see OrtApi::SessionOptionsAppendExecutionProvider() + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO */ typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus @@ -652,9 +647,9 @@ typedef struct OrtOpenVINOProviderOptions { const char* device_type; unsigned char enable_npu_fast_compile; const char* device_id; - size_t num_of_threads; ///< 0 = Use default number of threads - const char* cache_dir; ///< Any valid string path on the hardware target - void* context; ///< OpenCL Context + size_t num_of_threads; ///< 0 = Use default number of threads + const char* cache_dir; // path is set to empty by default + void* context; unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled } OrtOpenVINOProviderOptions;