From 3955dc1850b1b92c992fb20a8ab2a0828715c12a Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Wed, 4 Sep 2024 02:32:16 -0700 Subject: [PATCH 01/13] Implements blob compatibility check for NPU (#430) * Implements blob compatibility check for NPU * OVEP catches the NPU driver exception and return failure status * NPU to CPU fallback is disabled when inferencing with blob * Update NPU device exception handling approach * Changes failure status code to exception (std::runtime_error) * Capture all NPU related errors * Throw minimal error message with error type and error code for Release builds * Fix lint issues * Address review comments * Address review comments --------- Co-authored-by: Srirammaswamy --- .../providers/openvino/backend_manager.cc | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index be41b125e4440..4fca4037301fb 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -107,12 +108,15 @@ BackendManager::BackendManager(const GlobalContext& global_context, subgraph_context_, ep_ctx_handle_); } catch (const OnnxRuntimeException& ex) { + std::string exception_str = ex.what(); + bool eligible_for_cpu_fallback = device_type.find("NPU") != std::string::npos && + !GetGlobalContext().disable_cpu_fallback && + !ep_ctx_handle_.IsValidOVEPCtxGraph(); #if defined(OPENVINO_DISABLE_NPU_FALLBACK) - ORT_THROW(ex.what()); + eligible_for_cpu_fallback = false; #else - if (device_type.find("NPU") != std::string::npos && - !GetGlobalContext().disable_cpu_fallback) { - LOGS_DEFAULT(WARNING) << ex.what(); + if (eligible_for_cpu_fallback) { + LOGS_DEFAULT(VERBOSE) << exception_str; LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." << "Falling back to OV CPU for execution"; GetGlobalContext().device_type = "CPU"; @@ -125,10 +129,32 @@ BackendManager::BackendManager(const GlobalContext& global_context, } catch (std::string const& msg) { ORT_THROW(msg); } - } else { - ORT_THROW(ex.what()); } #endif + if (!eligible_for_cpu_fallback) { + if (device_type.find("NPU") != std::string::npos && + exception_str.find("intel_npu") != std::string::npos) { + // Handle NPU device related errors +#ifndef NDEBUG + ORT_THROW(exception_str + "\nModel needs to be recompiled\n"); +#else + std::string error_message = "UNKNOWN NPU ERROR"; + std::string error_code = "code 0x0"; + std::regex error_message_pattern(R"(\bZE_\w*\b)"); + std::regex error_code_pattern("code 0x[0-9a-fA-F]+"); + std::smatch matches; + if (std::regex_search(exception_str, matches, error_message_pattern)) { + error_message = matches[0]; + } + if (std::regex_search(exception_str, matches, error_code_pattern)) { + error_code = matches[0]; + } + throw std::runtime_error(error_message + ", " + error_code + "\nModel needs to be recompiled\n"); +#endif + } else { + ORT_THROW(exception_str); + } + } } } if (global_context_.export_ep_ctx_blob && !ep_ctx_handle_.IsValidOVEPCtxGraph()) { From b0a8bee8efc83254d46b9431bcb87b09417a6cee Mon Sep 17 00:00:00 2001 From: saurabh Date: Thu, 5 Sep 2024 18:30:37 +0530 Subject: [PATCH 02/13] Improvement in average inference latency for models running on OVEP NPU (#441) * Prototype shared memory allocator on Windows using OV-EP * Partially working allocator. Crashing on tensor destruction. Might have UMD exceptions. Needs further debug. Unknown if values are correct. * Hard code onnx perf to use RT NPU allocator for inputs * Fix allocation lookups coming from different level zero contexts * Page align OV allocation * Allocate input as WC * Only set tensors when they have changed. * Revert "Allocate input as WC" This reverts commit d43219f9b794d3ff86dd71df162932a44ab4ff42. * Hard code onnx perf to use RT NPU for outputs * Revert "Hard code onnx perf to use RT NPU for outputs" This reverts commit c1f3b3ec7caee7024b0ef887fa762d36001e0daa. * Hard code onnx perf to use RT NPU for outputs fixed * Fix onnx_perf_test app crash on tensor destroy * refactor: remove redundant ort_shape_to_ovshape lambda function * alocate buffer in NPU visible region from perf test application * remove redundant code * add command line parameter in perf test for using remote tensors * remove redundant code * remove redundant statements * fix crash during inference * remove redundant code * enable backward compatibility of remote tensor feature * Revert "enable backward compatibility of remote tensor feature" This reverts commit 1791b907651e3fca18d7257f97487e463a3303cb. * enable backward compatibility of remote tensor feature in OVEP --------- Co-authored-by: Javier E. Martinez Co-authored-by: Eric Crawford --- cmake/onnxruntime_providers_openvino.cmake | 4 + .../onnxruntime/core/framework/allocator.h | 2 + onnxruntime/core/framework/allocator.cc | 4 + .../openvino/backends/basic_backend.cc | 117 ++++++++++++++---- .../openvino/backends/basic_backend.h | 9 ++ .../openvino/openvino_execution_provider.cc | 17 +++ .../openvino/openvino_execution_provider.h | 4 +- .../core/providers/openvino/ov_allocator.cc | 55 ++++++++ .../core/providers/openvino/ov_allocator.h | 25 ++++ onnxruntime/test/perftest/ort_test_session.cc | 60 +++++++-- onnxruntime/test/perftest/ort_test_session.h | 3 + 11 files changed, 267 insertions(+), 33 deletions(-) create mode 100644 onnxruntime/core/providers/openvino/ov_allocator.cc create mode 100644 onnxruntime/core/providers/openvino/ov_allocator.h diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index e559583fae8f5..69805d60d4593 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -21,6 +21,10 @@ message(FATAL_ERROR "OpenVINO 2024.0 and newer are supported. Please, use latest OpenVINO release") endif() + if(OpenVINO_VERSION VERSION_GREATER_EQUAL 2024.4) + add_definitions(-DUSE_DEVICE_MEMORY=1) + endif() + if (WIN32) unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) endif() diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 097873c5e3653..abab118efd04f 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -50,6 +50,8 @@ constexpr const char* HIP = "Hip"; constexpr const char* HIP_PINNED = "HipPinned"; constexpr const char* OpenVINO_CPU = "OpenVINO_CPU"; constexpr const char* OpenVINO_GPU = "OpenVINO_GPU"; +constexpr const char* OpenVINO_RT = "OpenVINO_RT"; +constexpr const char* OpenVINO_RT_NPU = "OpenVINO_RT_NPU"; constexpr const char* WEBGPU_BUFFER = "WebGPU_Buffer"; constexpr size_t kAllocAlignment = 256; diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc index c3e96e450c59b..5e66f2b99fded 100644 --- a/onnxruntime/core/framework/allocator.cc +++ b/onnxruntime/core/framework/allocator.cc @@ -145,6 +145,10 @@ ORT_API_STATUS_IMPL(OrtApis::CreateMemoryInfo, _In_ const char* name1, enum OrtA *out = new OrtMemoryInfo( name1, type, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, static_cast(id1)), id1, mem_type1); + } else if (strcmp(name1, onnxruntime::OpenVINO_RT_NPU) == 0) { + *out = new OrtMemoryInfo( + name1, type, OrtDevice(OrtDevice::NPU, OrtDevice::MemType::DEFAULT, static_cast(id1)), id1, + mem_type1); } else if (strcmp(name1, onnxruntime::CUDA_PINNED) == 0) { *out = new OrtMemoryInfo( onnxruntime::CUDA_PINNED, type, OrtDevice(OrtDevice::CPU, OrtDevice::MemType::CUDA_PINNED, static_cast(id1)), diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 8d340e2daf4b5..df1a510c30209 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -48,14 +48,6 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr // Set the inference_num_threads property of the CPU SetNumThreads(device_config); -#ifndef NDEBUG - if (IsDebugEnabled()) { - std::string file_name = subgraph_context.subgraph_name + "_static.onnx"; - std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary); - model_proto->SerializeToOstream(outfile); - } -#endif - try { std::string dev_prec = global_context.device_type + "_" + global_context_.precision_str; @@ -295,16 +287,92 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - OVTensorPtr graph_input_blob; + auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_key; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_key = it->second; + } else { + // Does this make sense for both types of allocators? + auto input = graph_input_info.at(input_idx); + ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), + (void*)tensor.GetTensorRawData()); + if (allocator_name == OpenVINO_RT_NPU) { + ov_tensor_key.copy_needed = false; + } else { + ov_tensor_key.copy_needed = true; + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); + + try { + infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } + } + + if (ov_tensor_key.copy_needed) { + const char* ort_tensor_data = tensor.GetTensorData(); + size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; + std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); + } + } + input_idx++; + } + + // Set the output blob as remote blob + auto graph_output_info = exe_network_.Get().outputs(); + auto output_idx = 0; + for (auto output_info_iter = graph_output_info.begin(); + output_info_iter != graph_output_info.end(); ++output_info_iter) { + auto output_names = output_info_iter->get_names(); + std::string onnx_output_name; + std::string output_name; + bool output_name_found = false; + // using the output name retrieved from ONNX original to match with the output names returned by OV tensors + for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { + onnx_output_name = it->first; + if (output_names.find(onnx_output_name) != output_names.end()) { + // Assigning the output_name + output_name = it->first; + output_name_found = true; + break; + } + } + size_t batch_size = 1; + Ort::UnownedValue tensor = GetOutputTensor(context, + batch_size, + infer_request, + output_name, + subgraph_context_.output_names); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; + } else { + auto output = graph_output_info.at(output_idx); + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), + (void*)tensor.GetTensorRawData()); + if(allocator_name == OpenVINO_RT_NPU) { + ov_tensor_data.copy_needed = false; + } else { + ov_tensor_data.copy_needed = true; + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); + try { - graph_input_blob = infer_request->GetTensor(input_name); + infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); } catch (const char* msg) { ORT_THROW(msg); } - FillInputBlob(std::move(graph_input_blob), batch_slice_idx, std::move(input_name), context, subgraph_context_); } - input_idx++; + output_idx++; } + // Start Async inference infer_request->StartAsync(); } catch (const char* msg) { @@ -430,7 +498,6 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe auto graph_output_info = exe_network_.Get().outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { - OVTensorPtr graph_output_blob; auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; @@ -454,20 +521,24 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe " doesn't exist in the " "list of OpenVINO output tensor names"); } - try { - graph_output_blob = infer_request->GetTensor(output_name); - } catch (const char* msg) { - ORT_THROW(msg); - } + size_t batch_size = 1; Ort::UnownedValue output_tensor = GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); - auto mem_info = output_tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - return; + auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; } else { - size_t batch_slice = 0; - FillOutputBlob(std::move(graph_output_blob), output_tensor, batch_slice); + ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); + } + + if (ov_tensor_data.copy_needed) { + auto ort_tensor_data = output_tensor.GetTensorMutableData(); + size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; + std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); } } diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index cd242a06b27d4..4f430abe27b6b 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/openvino/contexts.h" @@ -20,6 +21,11 @@ namespace onnxruntime { namespace openvino_ep { +struct ov_tensor_data_t { + OVTensorPtr tensor_ptr; + bool copy_needed; +}; + class InferRequestsQueue; class BasicBackend : public IBackend { public: @@ -60,6 +66,9 @@ class BasicBackend : public IBackend { #if defined IO_BUFFER_ENABLED OVRemoteContextPtr remote_context_; #endif + + using ort_tensor_key_t = std::pair; + std::map ort_ov_tensor_map; }; class InferRequestsQueue { diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 29c45916795d3..4f9764212f37d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,6 +10,9 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" +#ifdef USE_DEVICE_MEMORY +#include "core/providers/openvino/ov_allocator.h" +#endif #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) @@ -180,4 +183,18 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } +#ifdef USE_DEVICE_MEMORY +std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { + AllocatorCreationInfo npu_allocator_info { + [this](OrtDevice::DeviceId device_id) { + return std::make_unique(global_context_->ie_core.Get(), OrtDevice::NPU, device_id, OpenVINO_RT_NPU); + }, + 0, + }; + + // fill in allocator + return std::vector{CreateAllocator(npu_allocator_info)}; +} +#endif + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 030e5bba71b67..42a8368a57e29 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,7 +189,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } - +#ifdef USE_DEVICE_MEMORY + std::vector CreatePreferredAllocators() override; +#endif private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc new file mode 100644 index 0000000000000..c7f22039a8b0e --- /dev/null +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -0,0 +1,55 @@ +// Copyright (C) Intel Corporation +// Licensed under the MIT License +#ifdef USE_DEVICE_MEMORY +#include "core/providers/openvino/ov_allocator.h" +#include "core/providers/openvino/ov_interface.h" +#include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +namespace onnxruntime { + +using namespace openvino_ep; + +constexpr size_t default_alignment = 4096; + +static inline size_t align_up(size_t size, size_t pow2_alignment) { + return (size + pow2_alignment - 1) & ~(pow2_alignment - 1); +} + +OVRTAllocator::OVRTAllocator(ov::Core& core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name) : IAllocator(OrtMemoryInfo(name, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(device_type, OrtDevice::MemType::DEFAULT, device_id), device_id, OrtMemTypeCPUInput)), core_(core) { + if (device_type == OrtDevice::NPU) { + remote_ctx_ = core_.get_default_context("NPU").as(); + } else { + ORT_THROW("Invalid device type"); + } +} + +void* OVRTAllocator::Alloc(size_t size) { + try { + size_t alloc_size = align_up(size + sizeof(ov::Tensor*) + default_alignment, default_alignment); + ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, + { alloc_size })); + uintptr_t data_ptr = reinterpret_cast(tensor->data()); + + ov::Tensor** ptr = reinterpret_cast(align_up(data_ptr + sizeof(ov::Tensor*), default_alignment)); + ptr[-1] = tensor; + + return reinterpret_cast(ptr); + + } catch (const ov::Exception& e) { + ORT_THROW(std::string("Alloc failed: ") + e.what()); + } + return nullptr; +} + +void OVRTAllocator::Free(void* p) { + try { + ov::Tensor** ptr = reinterpret_cast(p); + delete ptr[-1]; + } catch (const ov::Exception& e) { + ORT_THROW(std::string("Free failed: ") + e.what()); + } +} + +} // namespace onnxruntime +#endif diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h new file mode 100644 index 0000000000000..4a02f0013beac --- /dev/null +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -0,0 +1,25 @@ +// Copyright (C) Intel Corporation +// Licensed under the MIT License +#ifdef USE_DEVICE_MEMORY +#pragma once + +#include "core/common/inlined_containers.h" +#include "core/framework/allocator.h" +#include "openvino/runtime/remote_context.hpp" + + +namespace onnxruntime { + +class OVRTAllocator : public IAllocator { + public: + OVRTAllocator(ov::Core &core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); + void* Alloc(size_t size) override; + void Free(void* p) override; + + private: + ov::Core &core_; + ov::RemoteContext remote_ctx_; +}; + +} // namespace onnxruntime +#endif diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 837aeb3c37acd..48312e2c42e5d 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -34,10 +34,18 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { // Randomly pick one OrtValueArray from test_inputs_. (NOT ThreadSafe) const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); const size_t id = static_cast(dist_(rand_engine_, p)); + auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); - auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + + if (!use_device_mem) { + auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), output_names_raw_ptr.data(), output_names_raw_ptr.size()); + } else { + session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); + } + auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration duration_seconds = end - start; return duration_seconds; @@ -815,7 +823,12 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " "should be a boolean i.e. true or false. Default value is false.\n"); } - } else { + } else if (key == "use_device_mem") { + if (value == "true" || value == "True") { + use_device_mem = true; + } + } + else { ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } @@ -858,6 +871,27 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); input_names_str_[i] = m.GetInputName(i); input_names_[i] = input_names_str_[i].c_str(); } + + if (use_device_mem) { + Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + custom_allocator_ = std::make_unique(session_, memory_info); + for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { + Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + + std::vector output_shape = tensor_info.GetShape(); + + // free dimensions are treated as 1 if not overridden + for (int64_t& dim : output_shape) { + if (dim == -1) { + dim = 1; + } + } + + outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); + } + } } template @@ -944,9 +978,11 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { // iterate over all input nodes for (size_t i = 0; i < static_cast(input_length_); i++) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + if (!use_device_mem){ + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + } std::vector input_node_dim = tensor_info.GetShape(); // free dimensions are treated as 1 if not overridden @@ -955,12 +991,18 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { dim = 1; } } - - auto allocator = Ort::AllocatorWithDefaultOptions(); - Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(0, i, std::move(input_tensor)); + if (use_device_mem){ + Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(0, i, std::move(input_tensor)); + } else { + auto allocator = Ort::AllocatorWithDefaultOptions(); + Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(0, i, std::move(input_tensor)); + } } } return true; diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index f1a4220ab325e..e33041a2a0958 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -38,6 +38,8 @@ class OnnxRuntimeTestSession : public TestSession { std::mt19937 rand_engine_; std::uniform_int_distribution dist_; std::vector> test_inputs_; + std::unique_ptr custom_allocator_; + std::vector outputs_; std::vector output_names_; // The same size with output_names_. // TODO: implement a customized allocator, then we can remove output_names_ to simplify this code @@ -46,6 +48,7 @@ class OnnxRuntimeTestSession : public TestSession { std::vector input_names_str_; const int input_length_; std::string provider_name_; + bool use_device_mem = false; }; } // namespace perftest From df7febedb3610776f0745eb4d734c31d6d3d49f2 Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 Date: Wed, 4 Sep 2024 02:42:52 -0700 Subject: [PATCH 03/13] Disable driver caching for NPU when epctx enabled for ov version greater then 2024.3 --- .../core/providers/openvino/backends/basic_backend.cc | 5 +++++ onnxruntime/core/providers/openvino/ov_interface.h | 1 + 2 files changed, 6 insertions(+) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 8d340e2daf4b5..07d82f9f8db37 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -180,6 +180,11 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { device_property = std::make_pair("NPU_COMPILER_TYPE", env_npu_compiler_type); } device_config.emplace(ov::device::properties("NPU", device_property)); +#if (OPENVINO_VERSION_MAJOR >= 2024) && (OPENVINO_VERSION_MINOR > 3) + if (global_context_.export_ep_ctx_blob) { + global_context_.ie_core.Get().set_property("NPU", ov::intel_npu::bypass_umd_caching(true)); + } +#endif } } diff --git a/onnxruntime/core/providers/openvino/ov_interface.h b/onnxruntime/core/providers/openvino/ov_interface.h index fa22e0f3cb03d..f4da4ea3e3244 100644 --- a/onnxruntime/core/providers/openvino/ov_interface.h +++ b/onnxruntime/core/providers/openvino/ov_interface.h @@ -10,6 +10,7 @@ #include #include "openvino/openvino.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" #include "openvino/pass/convert_fp32_to_fp16.hpp" #include "openvino/frontend/manager.hpp" From 8259b03fc9bbc15999024b830575099d5b0b00ec Mon Sep 17 00:00:00 2001 From: saurabh Date: Thu, 12 Sep 2024 00:13:54 +0530 Subject: [PATCH 04/13] Ovep release lnl 1.2.1 (#445) * fix debug build issue and lint issues * change naming for OVEP NPU specific macro * fix unit tests and lint issues --- cmake/onnxruntime_providers_openvino.cmake | 2 +- .../openvino/backends/basic_backend.cc | 203 ++++++++++-------- .../openvino/backends/basic_backend.h | 2 +- .../openvino/openvino_execution_provider.cc | 12 +- .../openvino/openvino_execution_provider.h | 2 +- .../core/providers/openvino/ov_allocator.cc | 4 +- .../core/providers/openvino/ov_allocator.h | 9 +- onnxruntime/test/perftest/ort_test_session.cc | 21 +- 8 files changed, 142 insertions(+), 113 deletions(-) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 69805d60d4593..2eb3611bae902 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -22,7 +22,7 @@ endif() if(OpenVINO_VERSION VERSION_GREATER_EQUAL 2024.4) - add_definitions(-DUSE_DEVICE_MEMORY=1) + add_definitions(-DUSE_OVEP_NPU_MEMORY=1) endif() if (WIN32) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d274bbc64558d..1f9c61780f27a 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -292,90 +292,102 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_key; - ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_key = it->second; - } else { - // Does this make sense for both types of allocators? - auto input = graph_input_info.at(input_idx); - ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), - (void*)tensor.GetTensorRawData()); - if (allocator_name == OpenVINO_RT_NPU) { - ov_tensor_key.copy_needed = false; - } else { - ov_tensor_key.copy_needed = true; - } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); - + if ((global_context_.device_type.find("CPU") != std::string::npos || + global_context_.device_type.find("GPU") != std::string::npos)) { + OVTensorPtr graph_input_blob; try { - infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + graph_input_blob = infer_request->GetTensor(input_name); } catch (const char* msg) { ORT_THROW(msg); } - } - - if (ov_tensor_key.copy_needed) { - const char* ort_tensor_data = tensor.GetTensorData(); - size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; - std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); + FillInputBlob(std::move(graph_input_blob), batch_slice_idx, std::move(input_name), context, subgraph_context_); + } else { + auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_key; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_key = it->second; + } else { + // Does this make sense for both types of allocators? + auto input = graph_input_info.at(input_idx); + if (allocator_name == OpenVINO_RT_NPU) { + ov_tensor_key.copy_needed = false; + ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), + (void*)tensor.GetTensorRawData()); + } else { + ov_tensor_key.copy_needed = true; + ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape()); + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); + + if (ov_tensor_key.copy_needed) { + const char* ort_tensor_data = tensor.GetTensorData(); + size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; + std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); + } + + try { + infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } + } } } input_idx++; } - - // Set the output blob as remote blob - auto graph_output_info = exe_network_.Get().outputs(); - auto output_idx = 0; - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - auto output_names = output_info_iter->get_names(); - std::string onnx_output_name; - std::string output_name; - bool output_name_found = false; - // using the output name retrieved from ONNX original to match with the output names returned by OV tensors - for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { - onnx_output_name = it->first; - if (output_names.find(onnx_output_name) != output_names.end()) { - // Assigning the output_name - output_name = it->first; - output_name_found = true; - break; + if (global_context_.device_type.find("NPU") != std::string::npos) { + // Set the output blob as remote blob + auto graph_output_info = exe_network_.Get().outputs(); + auto output_idx = 0; + for (auto output_info_iter = graph_output_info.begin(); + output_info_iter != graph_output_info.end(); ++output_info_iter) { + auto output_names = output_info_iter->get_names(); + std::string onnx_output_name; + std::string output_name; + // using the output name retrieved from ONNX original to match with the output names returned by OV tensors + for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { + onnx_output_name = it->first; + if (output_names.find(onnx_output_name) != output_names.end()) { + // Assigning the output_name + output_name = it->first; + break; + } } - } - size_t batch_size = 1; - Ort::UnownedValue tensor = GetOutputTensor(context, - batch_size, - infer_request, - output_name, - subgraph_context_.output_names); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + size_t batch_size = 1; + Ort::UnownedValue tensor = GetOutputTensor(context, + batch_size, + infer_request, + output_name, + subgraph_context_.output_names); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_data; - ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; - } else { - auto output = graph_output_info.at(output_idx); - ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), - (void*)tensor.GetTensorRawData()); - if(allocator_name == OpenVINO_RT_NPU) { - ov_tensor_data.copy_needed = false; + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; } else { - ov_tensor_data.copy_needed = true; - } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); + auto output = graph_output_info.at(output_idx); + if (allocator_name == OpenVINO_RT_NPU) { + ov_tensor_data.copy_needed = false; + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), + (void*)tensor.GetTensorRawData()); + } else { + ov_tensor_data.copy_needed = true; + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape()); + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); - try { - infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); - } catch (const char* msg) { - ORT_THROW(msg); + try { + infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } } + output_idx++; } - output_idx++; } // Start Async inference @@ -503,6 +515,7 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe auto graph_output_info = exe_network_.Get().outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { + OVTensorPtr graph_output_blob; auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; @@ -526,24 +539,42 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe " doesn't exist in the " "list of OpenVINO output tensor names"); } - - size_t batch_size = 1; - Ort::UnownedValue output_tensor = - GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); - auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_data; - ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; + if ((global_context_.device_type.find("CPU") != std::string::npos || + global_context_.device_type.find("GPU") != std::string::npos)) { + try { + graph_output_blob = infer_request->GetTensor(output_name); + } catch (const char* msg) { + ORT_THROW(msg); + } + size_t batch_size = 1; + Ort::UnownedValue output_tensor = + GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); + auto mem_info = output_tensor.GetTensorMemoryInfo(); + if (mem_info.GetAllocatorName() == OpenVINO_GPU) { + return; + } else { + size_t batch_slice = 0; + FillOutputBlob(std::move(graph_output_blob), output_tensor, batch_slice); + } } else { - ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); - } + size_t batch_size = 1; + Ort::UnownedValue output_tensor = + GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); + auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; + } else { + ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); + } - if (ov_tensor_data.copy_needed) { - auto ort_tensor_data = output_tensor.GetTensorMutableData(); - size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; - std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); + if (ov_tensor_data.copy_needed) { + auto ort_tensor_data = output_tensor.GetTensorMutableData(); + size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; + std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); + } } } diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index 4f430abe27b6b..cd69e88f994b9 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -67,7 +67,7 @@ class BasicBackend : public IBackend { OVRemoteContextPtr remote_context_; #endif - using ort_tensor_key_t = std::pair; + using ort_tensor_key_t = std::pair; std::map ort_ov_tensor_map; }; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 4f9764212f37d..08144651319cf 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,7 +10,7 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #endif @@ -183,13 +183,13 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { - AllocatorCreationInfo npu_allocator_info { - [this](OrtDevice::DeviceId device_id) { + AllocatorCreationInfo npu_allocator_info{ + [this](OrtDevice::DeviceId device_id) { return std::make_unique(global_context_->ie_core.Get(), OrtDevice::NPU, device_id, OpenVINO_RT_NPU); - }, - 0, + }, + 0, }; // fill in allocator diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 42a8368a57e29..8b1c62c607f6e 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,7 +189,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY std::vector CreatePreferredAllocators() override; #endif private: diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index c7f22039a8b0e..6700244b754d8 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" @@ -28,7 +28,7 @@ void* OVRTAllocator::Alloc(size_t size) { try { size_t alloc_size = align_up(size + sizeof(ov::Tensor*) + default_alignment, default_alignment); ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, - { alloc_size })); + {alloc_size})); uintptr_t data_ptr = reinterpret_cast(tensor->data()); ov::Tensor** ptr = reinterpret_cast(align_up(data_ptr + sizeof(ov::Tensor*), default_alignment)); diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 4a02f0013beac..083cfc4d5aed3 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,24 +1,23 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #pragma once #include "core/common/inlined_containers.h" #include "core/framework/allocator.h" #include "openvino/runtime/remote_context.hpp" - namespace onnxruntime { class OVRTAllocator : public IAllocator { public: - OVRTAllocator(ov::Core &core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); + OVRTAllocator(ov::Core& core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); void* Alloc(size_t size) override; void Free(void* p) override; private: - ov::Core &core_; - ov::RemoteContext remote_ctx_; + ov::Core& core_; + ov::RemoteContext remote_ctx_; }; } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 48312e2c42e5d..ae7680571ced1 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -40,10 +40,10 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { if (!use_device_mem) { auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), output_names_raw_ptr.size()); + output_names_raw_ptr.data(), output_names_raw_ptr.size()); } else { - session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); + session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); } auto end = std::chrono::high_resolution_clock::now(); @@ -827,8 +827,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); if (value == "true" || value == "True") { use_device_mem = true; } - } - else { + } else { ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } @@ -888,8 +887,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } } - outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), - output_shape.size(), tensor_info.GetElementType())); + outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); } } } @@ -980,7 +979,7 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - if (!use_device_mem){ + if (!use_device_mem) { Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); } std::vector input_node_dim = tensor_info.GetShape(); @@ -991,15 +990,15 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { dim = 1; } } - if (use_device_mem){ + if (use_device_mem) { Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); + input_node_dim.size(), tensor_info.GetElementType()); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } else { auto allocator = Ort::AllocatorWithDefaultOptions(); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); + input_node_dim.size(), tensor_info.GetElementType()); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } From d69f6c935daba3730f4f70a87c0dd43eb4388595 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Thu, 5 Sep 2024 16:38:59 +0530 Subject: [PATCH 05/13] Add support to set session.workload_type in OVEP --- .../openvino/backends/basic_backend.cc | 12 +++++++++ .../openvino/backends/basic_backend.h | 1 + .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_execution_provider.cc | 1 + .../openvino/openvino_execution_provider.h | 6 +++-- .../openvino/openvino_provider_factory.cc | 26 +++++++++++++++---- .../core/session/provider_bridge_ort.cc | 2 ++ 7 files changed, 42 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 1f9c61780f27a..f70cbada43ff6 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -48,6 +48,9 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr // Set the inference_num_threads property of the CPU SetNumThreads(device_config); + //set workload type to decide on the performance mode + SetWorkLoadType(device_config); + try { std::string dev_prec = global_context.device_type + "_" + global_context_.precision_str; @@ -233,6 +236,15 @@ void BasicBackend::SetNumThreads(ov::AnyMap& device_config) { device_config.emplace(ov::inference_num_threads(global_context_.num_of_threads)); } +void BasicBackend::SetWorkLoadType(ov::AnyMap& device_config){ + if((global_context_.OpenVINO_Version.at(0) >= 2024 && + global_context_.OpenVINO_Version.at(1) >= 4 )){ + device_config.emplace(ov::workload_type(global_context_.workload_type)); + LOGS_DEFAULT(INFO) << log_tag << "Set workloadtype as " << global_context_.workload_type; + } +} + + // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on // an Infer Request indexed by infer_req_idx void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) { diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index cd69e88f994b9..ea44733974a1c 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -47,6 +47,7 @@ class BasicBackend : public IBackend { void EnableGPUThrottling(ov::AnyMap& device_config); void EnableStreams(); void SetNumThreads(ov::AnyMap& device_config); + void SetWorkLoadType(ov::AnyMap& device_config); void StartAsyncInference(Ort::KernelContext& context, std::shared_ptr infer_request); #ifdef IO_BUFFER_ENABLED diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 598e985676f8d..2024a95320161 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -32,6 +32,7 @@ struct GlobalContext { std::vector deviceAvailableList = {true, true, true, true, true, true, true, true}; std::string onnx_model_name; std::string onnx_model_path_name; + std::string workload_type; int onnx_opset_version; void* context = 0; bool use_api_2; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 08144651319cf..ac10b5bd6baa2 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -38,6 +38,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv global_context_->enable_qdq_optimizer = info.enable_qdq_optimizer_; global_context_->disable_cpu_fallback = info.disable_cpu_fallback_; global_context_->ep_context_embed_mode = info.so_epctx_embed_mode_; + global_context_->workload_type = info.workload_type_; // to check if target device is available // using ie_core capability GetAvailableDevices to fetch list of devices plugged in diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 8b1c62c607f6e..a94cff628a04a 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -91,6 +91,7 @@ struct OpenVINOExecutionProviderInfo { bool enable_qdq_optimizer_{false}; bool disable_cpu_fallback_{false}; bool so_epctx_embed_mode_{true}; + std::string workload_type_{""}; OpenVINOExecutionProviderInfo() = delete; @@ -100,7 +101,7 @@ struct OpenVINOExecutionProviderInfo { int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, bool enable_qdq_optimizer, bool disable_cpu_fallback, - bool so_epctx_embed_mode) + bool so_epctx_embed_mode, std::string workload_type) : precision_(std::move(precision)), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), @@ -113,7 +114,8 @@ struct OpenVINOExecutionProviderInfo { export_ep_ctx_blob_(export_ep_ctx_blob), enable_qdq_optimizer_(enable_qdq_optimizer), disable_cpu_fallback_(disable_cpu_fallback), - so_epctx_embed_mode_{so_epctx_embed_mode} { + so_epctx_embed_mode_{so_epctx_embed_mode}, + workload_type_(workload_type) { std::set ov_supported_device_types = {"CPU", "GPU", "GPU.0", "GPU.1", "NPU"}; diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 077ecc717502f..07e56e8213a30 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -15,7 +15,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, bool enable_qdq_optimizer, bool disable_cpu_fallback, - bool so_epctx_embed_mode) + bool so_epctx_embed_mode, + std::string workload_type) : precision_(precision), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), @@ -27,7 +28,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { export_ep_ctx_blob_(export_ep_ctx_blob), enable_qdq_optimizer_(enable_qdq_optimizer), disable_cpu_fallback_(disable_cpu_fallback), - so_epctx_embed_mode_(so_epctx_embed_mode) { + so_epctx_embed_mode_(so_epctx_embed_mode), + workload_type_(workload_type) { device_type_ = (device_type == nullptr) ? "" : device_type; cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; } @@ -52,14 +54,14 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { bool enable_qdq_optimizer_; bool disable_cpu_fallback_; bool so_epctx_embed_mode_; + std::string workload_type_; }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_, disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_, - disable_cpu_fallback_, - so_epctx_embed_mode_); + disable_cpu_fallback_, so_epctx_embed_mode_, workload_type_); return std::make_unique(info); } @@ -112,6 +114,8 @@ struct OpenVINO_Provider : Provider { bool so_epctx_embed_mode = true; + std::string workload_type = ""; + if (provider_options_map.find("device_type") != provider_options_map.end()) { device_type = provider_options_map.at("device_type").c_str(); @@ -318,6 +322,17 @@ struct OpenVINO_Provider : Provider { } } } + if (provider_options_map.find("workload_type") != provider_options_map.end()) { + workload_type = provider_options_map.at("workload_type"); + std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); + if (workload_type=="" || workload_type=="default") { + workload_type = "DEFAULT"; + } else if(workload_type=="efficient") { + workload_type = "EFFICIENT"; + } else { + ORT_THROW("[ERROR] [OpenVINO] Invalid workload_type - Supported modes are Default and Efficient \n"); + } + } return std::make_shared(const_cast(device_type.c_str()), const_cast(precision.c_str()), @@ -332,7 +347,8 @@ struct OpenVINO_Provider : Provider { export_ep_ctx_blob, enable_qdq_optimizer, disable_cpu_fallback, - so_epctx_embed_mode); + so_epctx_embed_mode, + workload_type); } void Initialize() override { diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 8e807c375143e..a08189f22b574 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1843,6 +1843,8 @@ void ORTSessionOptionsToOrtOpenVINOProviderOptions(ProviderOptions& ov_options, // defaults to true ov_options["so_epctx_embed_mode"] = "false"; } + + ov_options["workload_type"] = session_options->config_options.GetConfigOrDefault(kOrtSessionOptionsWorkloadType, "").c_str(); } std::shared_ptr OpenVINOProviderFactoryCreator::Create(ProviderOptions* provider_options_map, From c89cc2eaad96d9361563571c667c7422919b1530 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Tue, 10 Sep 2024 13:27:05 +0530 Subject: [PATCH 06/13] Add runtime option for workload type in OVEP --- .../openvino/backends/basic_backend.cc | 2 ++ .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_execution_provider.cc | 23 ++++++++++++++++++- .../openvino/openvino_execution_provider.h | 3 +++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index f70cbada43ff6..157104bd66122 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -403,6 +403,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } // Start Async inference + exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); @@ -511,6 +512,7 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe } // Start Async inference + exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 2024a95320161..348d4a20094fb 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -33,6 +33,7 @@ struct GlobalContext { std::string onnx_model_name; std::string onnx_model_path_name; std::string workload_type; + std::string runtime_workload_type; int onnx_opset_version; void* context = 0; bool use_api_2; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index ac10b5bd6baa2..5d11f20175713 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -13,6 +13,7 @@ #ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #endif +#include "core/session/onnxruntime_run_options_config_keys.h" #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) @@ -180,10 +181,29 @@ common::Status OpenVINOExecutionProvider::Compile( }; node_compute_funcs.push_back(compute_info); } - return Status::OK(); } +common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptions& run_options) { + std::string workload_type=""; + auto workload_type_opt = run_options.GetConfigOptions().GetConfigEntry(kOrtRunOptionsWorkloadType); + if(workload_type_opt.has_value()){ + workload_type = workload_type_opt.value(); + } + std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); + if (workload_type=="" || workload_type=="default") { + workload_type = "DEFAULT"; + } else if(workload_type=="efficient") { + workload_type = "EFFICIENT"; + } else { + ORT_THROW("[ERROR] [OpenVINO] Invalid workload_type - Supported modes are Default and Efficient \n"); + } + global_context_->runtime_workload_type = workload_type; + return Status::OK(); +} +common::Status OpenVINOExecutionProvider::OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& run_options) { + return Status::OK(); + } #ifdef USE_OVEP_NPU_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info{ @@ -198,4 +218,5 @@ std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() } #endif + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index a94cff628a04a..10ea4c4af501b 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -187,6 +187,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider { Status Compile(const std::vector& fused_nodes, std::vector& node_compute_funcs) override; + Status OnRunStart(const onnxruntime::RunOptions& run_options) override; + + Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override; const void* GetExecutionHandle() const noexcept override { return nullptr; From a49cbf6bd7385e4576fa4bbe8a5a9fb9b4c42316 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Wed, 11 Sep 2024 15:15:35 +0530 Subject: [PATCH 07/13] Set property for worklod type as anymap property --- .../core/providers/openvino/backends/basic_backend.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 157104bd66122..354452fa7a47c 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -239,7 +239,9 @@ void BasicBackend::SetNumThreads(ov::AnyMap& device_config) { void BasicBackend::SetWorkLoadType(ov::AnyMap& device_config){ if((global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) >= 4 )){ - device_config.emplace(ov::workload_type(global_context_.workload_type)); + std::pair device_property; + device_property = std::make_pair("WORKLOAD_TYPE", global_context_.workload_type); + device_config.emplace(ov::device::properties("NPU", device_property)); LOGS_DEFAULT(INFO) << log_tag << "Set workloadtype as " << global_context_.workload_type; } } @@ -403,7 +405,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } // Start Async inference - exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); + exe_network_.Get().set_property({{"WORKLOAD_TYPE", global_context_.runtime_workload_type}}); infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); @@ -512,7 +514,7 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe } // Start Async inference - exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); + exe_network_.Get().set_property({{"WORKLOAD_TYPE", global_context_.runtime_workload_type}}); infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); From d31050b4f380befabec68766fd7ba86a34e5f3f0 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Thu, 19 Sep 2024 11:50:54 +0530 Subject: [PATCH 08/13] Fix reference to global_context to be set during run_option --- .../openvino/backends/basic_backend.cc | 10 ++++++-- .../openvino/openvino_execution_provider.cc | 24 +++++++++---------- .../openvino/openvino_execution_provider.h | 1 + 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 354452fa7a47c..cc7af183ee3bb 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -405,7 +405,10 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } // Start Async inference - exe_network_.Get().set_property({{"WORKLOAD_TYPE", global_context_.runtime_workload_type}}); + // exe_network_.Get().set_property(ov::device::properties("NPU", + // {{"WORKLOAD_TYPE", global_context_.runtime_workload_type}})); + std::cout << " Global context runtime workload type = " << global_context_.runtime_workload_type << std::endl; + exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); @@ -514,7 +517,10 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe } // Start Async inference - exe_network_.Get().set_property({{"WORKLOAD_TYPE", global_context_.runtime_workload_type}}); + // exe_network_.Get().set_property(ov::device::properties("NPU", + // {{"WORKLOAD_TYPE", global_context_.runtime_workload_type}})); + exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); + infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 5d11f20175713..aad22264ecaed 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -151,7 +151,7 @@ common::Status OpenVINOExecutionProvider::Compile( graph_body_viewer, *GetLogger(), ep_ctx_handle_); - + backend_manager_ = backend_manager; compute_info.create_state_func = [backend_manager](ComputeContext* context, FunctionState* state) { OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState(); @@ -184,20 +184,20 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptions& run_options) { - std::string workload_type=""; + // std::string workload_type=""; auto workload_type_opt = run_options.GetConfigOptions().GetConfigEntry(kOrtRunOptionsWorkloadType); if(workload_type_opt.has_value()){ - workload_type = workload_type_opt.value(); + std::string workload_type = workload_type_opt.value(); + std::cout << " Workload type from RunOption = " << workload_type << std::endl; + std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); + if (workload_type=="default") { + global_context_->runtime_workload_type = "DEFAULT"; + // backend_manager_->GetGlobalContext().runtime_workload_type = "DEFAULT"; + } else if(workload_type=="efficient") { + global_context_->runtime_workload_type = "EFFICIENT"; + // backend_manager_->GetGlobalContext().runtime_workload_type = "EFFICIENT"; + } } - std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); - if (workload_type=="" || workload_type=="default") { - workload_type = "DEFAULT"; - } else if(workload_type=="efficient") { - workload_type = "EFFICIENT"; - } else { - ORT_THROW("[ERROR] [OpenVINO] Invalid workload_type - Supported modes are Default and Efficient \n"); - } - global_context_->runtime_workload_type = workload_type; return Status::OK(); } diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 10ea4c4af501b..ac2b13503ed2d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -197,6 +197,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { #ifdef USE_OVEP_NPU_MEMORY std::vector CreatePreferredAllocators() override; #endif + std::shared_ptr backend_manager_; private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; From 541afee92debc2b05af36809cf6ab49508772cb1 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Thu, 19 Sep 2024 17:30:31 +0530 Subject: [PATCH 09/13] Pass global_context by reference to backend_manager --- .../providers/openvino/backend_manager.cc | 66 ++++----- .../core/providers/openvino/backend_manager.h | 6 +- .../core/providers/openvino/backend_utils.cc | 16 +-- .../core/providers/openvino/backend_utils.h | 4 +- .../openvino/backends/backend_factory.cc | 4 +- .../openvino/backends/basic_backend.cc | 128 +++++++++--------- .../openvino/backends/basic_backend.h | 4 +- .../core/providers/openvino/ibackend.h | 2 +- .../openvino/openvino_execution_provider.cc | 7 +- 9 files changed, 115 insertions(+), 122 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index 4fca4037301fb..7aa75cba3e87f 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -21,19 +21,19 @@ namespace onnxruntime { namespace openvino_ep { -GlobalContext& BackendManager::GetGlobalContext() { +GlobalContext* BackendManager::GetGlobalContext() { return global_context_; } -BackendManager::BackendManager(const GlobalContext& global_context, +BackendManager::BackendManager(GlobalContext* global_context, const onnxruntime::Node& fused_node, const onnxruntime::GraphViewer& subgraph, const logging::Logger& logger, EPCtxHandler& ep_ctx_handle_) { global_context_ = global_context; - openvino_sdk_version_ = std::to_string(global_context_.OpenVINO_Version.at(0)) + "." + - std::to_string(global_context_.OpenVINO_Version.at(1)); + openvino_sdk_version_ = std::to_string(global_context_->OpenVINO_Version.at(0)) + "." + + std::to_string(global_context_->OpenVINO_Version.at(1)); if (ep_ctx_handle_.CheckForOVEPCtxNode(subgraph, openvino_sdk_version_)) { if (ep_ctx_handle_.ImportBlobFromEPCtxModel(subgraph) != Status::OK()) ORT_THROW("Import blob from model failed"); @@ -66,17 +66,17 @@ BackendManager::BackendManager(const GlobalContext& global_context, } subgraph_context_.subgraph_name = fused_node.Name(); auto model_proto = GetModelProtoFromFusedNode(fused_node, subgraph, logger); - std::string device_type = openvino_ep::BackendManager::GetGlobalContext().device_type; + std::string device_type = openvino_ep::BackendManager::GetGlobalContext()->device_type; if (ModelHasSymbolicInputDims(subgraph)) { subgraph_context_.has_dynamic_input_shape = true; LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has symbolic input dims"; - ORT_ENFORCE(!global_context_.enable_qdq_optimizer, + ORT_ENFORCE(!global_context_->enable_qdq_optimizer, "QDQ stripping should not be enabled for models with dynamic input shapes. " "Set enable_qdq_optimizer to False"); - if ((GetGlobalContext().device_type.find("CPU") != std::string::npos || - GetGlobalContext().device_type.find("GPU") != std::string::npos) && - !GetGlobalContext().disable_dynamic_shapes) { + if ((GetGlobalContext()->device_type.find("CPU") != std::string::npos || + GetGlobalContext()->device_type.find("GPU") != std::string::npos) && + !GetGlobalContext()->disable_dynamic_shapes) { LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. " << "Creating backend Dynamic Shapes"; try { @@ -110,7 +110,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, } catch (const OnnxRuntimeException& ex) { std::string exception_str = ex.what(); bool eligible_for_cpu_fallback = device_type.find("NPU") != std::string::npos && - !GetGlobalContext().disable_cpu_fallback && + !GetGlobalContext()->disable_cpu_fallback && !ep_ctx_handle_.IsValidOVEPCtxGraph(); #if defined(OPENVINO_DISABLE_NPU_FALLBACK) eligible_for_cpu_fallback = false; @@ -119,8 +119,8 @@ BackendManager::BackendManager(const GlobalContext& global_context, LOGS_DEFAULT(VERBOSE) << exception_str; LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." << "Falling back to OV CPU for execution"; - GetGlobalContext().device_type = "CPU"; - GetGlobalContext().precision_str = "FP32"; + GetGlobalContext()->device_type = "CPU"; + GetGlobalContext()->precision_str = "FP32"; try { concrete_backend_ = BackendFactory::MakeBackend(model_proto, GetGlobalContext(), @@ -157,7 +157,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, } } } - if (global_context_.export_ep_ctx_blob && !ep_ctx_handle_.IsValidOVEPCtxGraph()) { + if (global_context_->export_ep_ctx_blob && !ep_ctx_handle_.IsValidOVEPCtxGraph()) { auto status = onnxruntime::openvino_ep::BackendManager::ExportCompiledBlobAsEPCtxNode(subgraph, logger); if ((!status.IsOK())) { @@ -172,7 +172,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, // the EPContext node. Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& graph_body_viewer, const logging::Logger& logger) { - if (GetGlobalContext().disable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape) { + if (GetGlobalContext()->disable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape) { std::string exception_str = "Exporting dynamically compiled models at runtime is not supported. " "Cannot export blobs of dynamic models that request static shape inference. " @@ -184,19 +184,19 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie auto compiled_model = concrete_backend_->GetOVCompiledModel(); std::string graph_name = ""; // Epctx file path from SO is mapped to cache_dir variable for OVEP for readability - if (!global_context_.cache_dir.empty()) { - graph_name = global_context_.cache_dir; + if (!global_context_->cache_dir.empty()) { + graph_name = global_context_->cache_dir; } else { - graph_name = global_context_.onnx_model_path_name; + graph_name = global_context_->onnx_model_path_name; // Remove extension so we can append suffix to form the complete name of output graph - size_t dot = global_context_.onnx_model_path_name.find_last_of("."); + size_t dot = global_context_->onnx_model_path_name.find_last_of("."); graph_name = graph_name.substr(0, dot); if (dot != std::string::npos) graph_name += "_ctx.onnx"; } // If embed_mode, then pass on the serialized blob // If not embed_mode, dump the blob here and only pass on the path to the blob - if (global_context_.ep_context_embed_mode) { + if (global_context_->ep_context_embed_mode) { std::ostringstream model_blob_stream; compiled_model.export_model(model_blob_stream); model_blob_str = std::move(model_blob_stream).str(); @@ -218,7 +218,7 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie ORT_RETURN_IF_ERROR(ep_ctx_handle_.ExportEPCtxModel(graph_body_viewer, graph_name, logger, - global_context_.ep_context_embed_mode, + global_context_->ep_context_embed_mode, std::move(model_blob_str), openvino_sdk_version_)); @@ -337,8 +337,8 @@ BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node, }; // QDQ stripping enabled only for the NPU - if (global_context_.device_type.find("NPU") != std::string::npos && - global_context_.enable_qdq_optimizer && + if (global_context_->device_type.find("NPU") != std::string::npos && + global_context_->enable_qdq_optimizer && IsQDQGraph(subgraph)) { LOGS_DEFAULT(INFO) << "[OpenVINO-EP] QDQ optimization pass status: 1"; std::unique_ptr model; @@ -346,7 +346,7 @@ BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node, auto model_proto = model->ToProto(); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); print_model_proto_duration(); - DumpOpenVINOEPModel(global_context_.onnx_model_path_name, model_proto.get(), fused_node); + DumpOpenVINOEPModel(global_context_->onnx_model_path_name, model_proto.get(), fused_node); ORT_ENFORCE(status.IsOK(), status.ErrorMessage()); return model_proto; } else { @@ -356,7 +356,7 @@ BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node, model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); subgraph.ToProto(*model_proto->mutable_graph(), true, true); print_model_proto_duration(); - DumpOpenVINOEPModel(global_context_.onnx_model_path_name, model_proto.get(), fused_node); + DumpOpenVINOEPModel(global_context_->onnx_model_path_name, model_proto.get(), fused_node); return model_proto; } } @@ -448,13 +448,13 @@ void BackendManager::Compute(OrtKernelContext* context) { // by rewriting the model to static shaped model at runtime based on input shape. // disable_dynamic_shapes is always set to true for OV NPU plugin. if (subgraph_context_.has_dynamic_input_shape && - !GetGlobalContext().disable_dynamic_shapes && - (GetGlobalContext().device_type.find("CPU") != std::string::npos || - GetGlobalContext().device_type.find("GPU") != std::string::npos)) { + !GetGlobalContext()->disable_dynamic_shapes && + (GetGlobalContext()->device_type.find("CPU") != std::string::npos || + GetGlobalContext()->device_type.find("GPU") != std::string::npos)) { concrete_backend_->Infer(context); } else if (subgraph_context_.has_dynamic_input_shape) { std::vector> tensor_shapes = GetInputTensorShapes(ctx); - auto key = MakeMapKeyString(tensor_shapes, GetGlobalContext().device_type); + auto key = MakeMapKeyString(tensor_shapes, GetGlobalContext()->device_type); std::shared_ptr dynamic_backend; auto search = backend_map_.find(key); if (search == backend_map_.end()) { @@ -474,14 +474,14 @@ void BackendManager::Compute(OrtKernelContext* context) { LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU."; ORT_THROW(ex.what()); #else - if (GetGlobalContext().device_type.find("NPU") != std::string::npos && - !GetGlobalContext().disable_cpu_fallback) { + if (GetGlobalContext()->device_type.find("NPU") != std::string::npos && + !GetGlobalContext()->disable_cpu_fallback) { LOGS_DEFAULT(WARNING) << ex.what(); LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." << "Falling back to OV CPU for execution"; - GetGlobalContext().device_type = "CPU"; - GetGlobalContext().precision_str = "FP32"; - key = MakeMapKeyString(tensor_shapes, GetGlobalContext().device_type); + GetGlobalContext()->device_type = "CPU"; + GetGlobalContext()->precision_str = "FP32"; + key = MakeMapKeyString(tensor_shapes, GetGlobalContext()->device_type); try { dynamic_backend = BackendFactory::MakeBackend(modelproto_with_concrete_shapes, GetGlobalContext(), diff --git a/onnxruntime/core/providers/openvino/backend_manager.h b/onnxruntime/core/providers/openvino/backend_manager.h index b9ff7a72372b3..578c1c199f832 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.h +++ b/onnxruntime/core/providers/openvino/backend_manager.h @@ -19,7 +19,7 @@ namespace openvino_ep { // Singleton class that manages all the backends class BackendManager { public: - BackendManager(const GlobalContext& global_context, + BackendManager(GlobalContext* global_context, const onnxruntime::Node& fused_node, const onnxruntime::GraphViewer& subgraph, const logging::Logger& logger, @@ -27,7 +27,7 @@ class BackendManager { void Compute(OrtKernelContext* context); void ShutdownBackendManager(); void SetGlobalCotext(const GlobalContext& global_context); - GlobalContext& GetGlobalContext(); + GlobalContext* GetGlobalContext(); Status ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& subgraph, const logging::Logger& logger); @@ -51,7 +51,7 @@ class BackendManager { std::shared_ptr concrete_backend_; std::map> backend_map_; SubGraphContext subgraph_context_; - GlobalContext global_context_; + GlobalContext* global_context_; EPCtxHandler ep_ctx_handle_{}; std::string openvino_sdk_version_{}; }; diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index f772b9c3b0478..97b0d4eb74883 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -40,17 +40,17 @@ struct static_cast_int64 { }; std::shared_ptr -CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, +CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext* global_context, std::map>& const_outputs_map) { if (IsCILogEnabled()) { std::cout << "CreateNgraphFunc" << std::endl; } const std::string model = model_proto.SerializeAsString(); try { - auto cnn_network = global_context.ie_core.ReadModel(model, global_context.onnx_model_path_name); + auto cnn_network = global_context->ie_core.ReadModel(model, global_context->onnx_model_path_name); // Check for Constant Folding - if (!global_context.is_wholly_supported_graph) { + if (!global_context->is_wholly_supported_graph) { ov::pass::ConstantFolding pass_const_obj; pass_const_obj.run_on_model(cnn_network); auto& results = const_cast(cnn_network.get()->get_results()); @@ -129,13 +129,13 @@ GetOutputTensor(Ort::KernelContext& context, return context.GetOutput(index, output_shape.get(), num_dims); } -int GetFirstAvailableDevice(GlobalContext& global_context) { +int GetFirstAvailableDevice(GlobalContext* global_context) { int i = 0; // Get the first available VAD-M device and set the device to busy while (i < 8) { - bool device = global_context.deviceAvailableList[i]; + bool device = global_context->deviceAvailableList[i]; if (device) { - global_context.deviceAvailableList[i] = false; + global_context->deviceAvailableList[i] = false; break; } i++; @@ -144,9 +144,9 @@ int GetFirstAvailableDevice(GlobalContext& global_context) { // make all remaining devices free if (i == 8) { i = 0; - global_context.deviceAvailableList[i] = false; + global_context->deviceAvailableList[i] = false; for (int j = 1; j < 8; j++) { - global_context.deviceAvailableList[j] = true; + global_context->deviceAvailableList[j] = true; } } return i; diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h index 9e65770da7d23..a98ff27426a92 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.h +++ b/onnxruntime/core/providers/openvino/backend_utils.h @@ -34,7 +34,7 @@ bool IsDebugEnabled(); // Internal diagnostic function. bool IsCILogEnabled(); -int GetFirstAvailableDevice(GlobalContext& global_context); +int GetFirstAvailableDevice(GlobalContext* global_context); void FillOutputsWithConstantData(std::shared_ptr node, Ort::UnownedValue& out_tensor); @@ -62,7 +62,7 @@ void FillOutputBlob(OVTensorPtr outputBlob, Ort::UnownedValue& output_tensor, std::shared_ptr CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, - const GlobalContext& global_context, + const GlobalContext* global_context, std::map>& const_outputs_map); void printPerformanceCounts(const std::vector& performanceMap, diff --git a/onnxruntime/core/providers/openvino/backends/backend_factory.cc b/onnxruntime/core/providers/openvino/backends/backend_factory.cc index b7e4aed6e7e18..88410a8f75e88 100644 --- a/onnxruntime/core/providers/openvino/backends/backend_factory.cc +++ b/onnxruntime/core/providers/openvino/backends/backend_factory.cc @@ -12,10 +12,10 @@ namespace openvino_ep { std::shared_ptr BackendFactory::MakeBackend(std::unique_ptr& model_proto, - GlobalContext& global_context, + GlobalContext* global_context, const SubGraphContext& subgraph_context, EPCtxHandler& ep_ctx_handle) { - std::string type = global_context.device_type; + std::string type = global_context->device_type; if (type == "CPU" || type.find("GPU") != std::string::npos || type.find("NPU") != std::string::npos || type.find("HETERO") != std::string::npos || diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index cc7af183ee3bb..b2c048381a62b 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -21,11 +21,11 @@ namespace openvino_ep { using namespace backend_utils; BasicBackend::BasicBackend(std::unique_ptr& model_proto, - GlobalContext& global_context, + GlobalContext* global_context, const SubGraphContext& subgraph_context, EPCtxHandler& ep_ctx_handle) : global_context_(global_context), subgraph_context_(subgraph_context) { - std::string& hw_target = global_context_.device_type; + std::string& hw_target = global_context_->device_type; is_ep_ctx_graph_ = ep_ctx_handle.IsValidOVEPCtxGraph(); @@ -52,40 +52,39 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr SetWorkLoadType(device_config); try { - std::string dev_prec = global_context.device_type + "_" + global_context_.precision_str; + std::string dev_prec = global_context_->device_type + "_" + global_context_->precision_str; - if (global_context.is_wholly_supported_graph) { // Full graph is supported + if (global_context_->is_wholly_supported_graph) { // Full graph is supported #if defined(IO_BUFFER_ENABLED) if (is_ep_ctx_graph_) { std::istringstream model_stream(ep_ctx_handle.GetModelBlobString()); - exe_network_ = global_context_.ie_core.ImportModel(model_stream, + exe_network_ = global_context_->ie_core.ImportModel(model_stream, remote_context_, subgraph_context_.subgraph_name); - } else if ((global_context.device_type.find("GPU") != std::string::npos) && - (global_context_.context != nullptr)) { + } else if ((global_context_->device_type.find("GPU") != std::string::npos) && + (global_context_->context != nullptr)) { LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled"; - cl_context ctx = static_cast(global_context_.context); - remote_context_ = new ov::intel_gpu::ocl::ClContext(global_context_.ie_core.Get(), ctx); + cl_context ctx = static_cast(global_context_->context); + remote_context_ = new ov::intel_gpu::ocl::ClContext(global_context_->ie_core.Get(), ctx); ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); - exe_network_ = global_context_.ie_core.CompileModel( + exe_network_ = global_context_->ie_core.CompileModel( ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name); } else { ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); - exe_network_ = global_context_.ie_core.CompileModel( + exe_network_ = global_context_->ie_core.CompileModel( ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name); } #else // !IO_BUFFER_ENABLED - std::string prec_str = (global_context_.precision_str != "ACCURACY") ? global_context_.precision_str : global_context_.model_precision; + std::string prec_str = (global_context_->precision_str != "ACCURACY") ? global_context_->precision_str : global_context_->model_precision; if (is_ep_ctx_graph_) { // If the blob is held in an EPContext node, then skip FE+Compile // and directly move on to creating a backend with the executable blob - exe_network_ = global_context_.ie_core.ImportModel(ep_ctx_handle.GetModelBlobStream(), + exe_network_ = global_context_->ie_core.ImportModel(ep_ctx_handle.GetModelBlobStream(), hw_target, device_config, - global_context_.ep_context_embed_mode, + global_context_->ep_context_embed_mode, subgraph_context_.subgraph_name); - ie_cnn_network_ = exe_network_.Get().get_runtime_model(); - } else if (global_context_.export_ep_ctx_blob && + } else if (global_context_->export_ep_ctx_blob && hw_target.find("NPU") != std::string::npos) { std::shared_ptr ov_model; { @@ -93,28 +92,28 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr if (!subgraph_context.has_dynamic_input_shape) { delete model_proto.release(); } - ov_model = global_context_.ie_core.Get().read_model(model, ov::Tensor()); + ov_model = global_context_->ie_core.Get().read_model(model, ov::Tensor()); } - exe_network_ = OVExeNetwork(global_context_.ie_core.Get().compile_model(ov_model, hw_target, device_config)); + exe_network_ = OVExeNetwork(global_context_->ie_core.Get().compile_model(ov_model, hw_target, device_config)); } else if ((!subgraph_context_.has_dynamic_input_shape) && ((hw_target.find("AUTO") == std::string::npos) || - (global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) > 2))) { + (global_context_->OpenVINO_Version.at(0) >= 2024 && global_context_->OpenVINO_Version.at(1) > 2))) { // Optimized OV compile_model API is supported with AUTO from version 2024.3 and above // Inputs with static dimenstions const std::string model = model_proto->SerializeAsString(); - exe_network_ = global_context_.ie_core.CompileModel(model, + exe_network_ = global_context_->ie_core.CompileModel(model, hw_target, device_config, subgraph_context_.subgraph_name); } else { // For all other types use ov::Model Type ie_cnn_network_ = CreateOVModel(*model_proto, global_context_, const_outputs_map_); - exe_network_ = global_context_.ie_core.CompileModel( + exe_network_ = global_context_->ie_core.CompileModel( ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name); } #endif } else { // Full graph is not supported ie_cnn_network_ = CreateOVModel(*model_proto, global_context_, const_outputs_map_); - exe_network_ = global_context_.ie_core.CompileModel( + exe_network_ = global_context_->ie_core.CompileModel( ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name); } LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; @@ -138,21 +137,21 @@ bool BasicBackend::ValidateSubgraph(std::mapprecision_str.find("FP16") != std::string::npos && + global_context_->device_type == "GPU") { device_config.emplace(ov::hint::inference_precision("f16")); } - if (global_context_.precision_str.find("FP32") != std::string::npos) { + if (global_context_->precision_str.find("FP32") != std::string::npos) { device_config.emplace(ov::hint::inference_precision("f32")); } - if (global_context_.precision_str.find("ACCURACY") != std::string::npos && - global_context_.device_type == "GPU") { - if (global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) >= 1) { + if (global_context_->precision_str.find("ACCURACY") != std::string::npos && + global_context_->device_type == "GPU") { + if (global_context_->OpenVINO_Version.at(0) >= 2024 && global_context_->OpenVINO_Version.at(1) >= 1) { device_config.emplace(ov::hint::inference_precision(ov::element::undefined)); device_config.emplace(ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); } else { - if (global_context_.model_precision != "") - device_config.emplace(ov::hint::inference_precision(global_context_.model_precision)); + if (global_context_->model_precision != "") + device_config.emplace(ov::hint::inference_precision(global_context_->model_precision)); } } #ifndef NDEBUG @@ -163,10 +162,10 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { // Set a priority level for the current workload for preemption; default priority is "DEFAULT" // CPU Plugin doesn't support workload priority - if (global_context_.device_type.find("CPU") == std::string::npos) - device_config.emplace(ov::hint::model_priority(global_context_.model_priority)); + if (global_context_->device_type.find("CPU") == std::string::npos) + device_config.emplace(ov::hint::model_priority(global_context_->model_priority)); - if (global_context_.device_type.find("NPU") != std::string::npos) { + if (global_context_->device_type.find("NPU") != std::string::npos) { std::pair device_property; device_property = std::make_pair("NPU_COMPILER_TYPE", "DRIVER"); @@ -187,21 +186,21 @@ void BasicBackend::EnableCaching(ov::AnyMap& device_config) { // cache_dir argument has no effect when working with an embed-mode EPContext Graph if (is_ep_ctx_graph_) return; - if (!global_context_.cache_dir.empty() && !global_context_.export_ep_ctx_blob) { + if (!global_context_->cache_dir.empty() && !global_context_->export_ep_ctx_blob) { LOGS_DEFAULT(INFO) << log_tag << "Enables Caching"; - if (global_context_.device_type.find("AUTO:GPU") != std::string::npos) { + if (global_context_->device_type.find("AUTO:GPU") != std::string::npos) { std::pair device_property; - device_property = std::make_pair("CACHE_DIR", global_context_.cache_dir); + device_property = std::make_pair("CACHE_DIR", global_context_->cache_dir); device_config.emplace(ov::device::properties("GPU", device_property)); } else { - global_context_.ie_core.SetCache(global_context_.cache_dir); + global_context_->ie_core.SetCache(global_context_->cache_dir); } } } void BasicBackend::EnableGPUThrottling(ov::AnyMap& device_config) { - if (global_context_.enable_opencl_throttling == true && - global_context_.device_type.find("GPU") != std::string::npos) { + if (global_context_->enable_opencl_throttling == true && + global_context_->device_type.find("GPU") != std::string::npos) { LOGS_DEFAULT(INFO) << log_tag << "Enabled OpenCL queue throttling for GPU device"; std::pair device_property; device_property = std::make_pair("PLUGIN_THROTTLE", "1"); @@ -212,37 +211,37 @@ void BasicBackend::EnableGPUThrottling(ov::AnyMap& device_config) { void BasicBackend::EnableStreams() { // Return silently for NPU as it's currently treated as a read-only flag by the NPU plugin // and throws an exception for the same - if (global_context_.device_type.find("NPU") != std::string::npos) + if (global_context_->device_type.find("NPU") != std::string::npos) return; // Streams can be set only if the device is not one of AUTO, MULTI, or HETERO // Throw an exception if the user tries to set num_streams for these devices - if ((global_context_.device_type.find("MULTI") != std::string::npos) || - (global_context_.device_type.find("HETERO") != std::string::npos) || - (global_context_.device_type.find("AUTO") != std::string::npos)) { - if (global_context_.num_streams != 1) { + if ((global_context_->device_type.find("MULTI") != std::string::npos) || + (global_context_->device_type.find("HETERO") != std::string::npos) || + (global_context_->device_type.find("AUTO") != std::string::npos)) { + if (global_context_->num_streams != 1) { ORT_THROW(log_tag + "Cannot set NUM_STREAMS to " + - std::to_string(global_context_.num_streams) + " for device " + global_context_.device_type); + std::to_string(global_context_->num_streams) + " for device " + global_context_->device_type); } // Do nothing } else { - global_context_.ie_core.SetStreams(global_context_.device_type, global_context_.num_streams); + global_context_->ie_core.SetStreams(global_context_->device_type, global_context_->num_streams); } } void BasicBackend::SetNumThreads(ov::AnyMap& device_config) { // inference_num_threads is applicable only for the CPU device - if (global_context_.device_type.find("CPU") != std::string::npos) - device_config.emplace(ov::inference_num_threads(global_context_.num_of_threads)); + if (global_context_->device_type.find("CPU") != std::string::npos) + device_config.emplace(ov::inference_num_threads(global_context_->num_of_threads)); } void BasicBackend::SetWorkLoadType(ov::AnyMap& device_config){ - if((global_context_.OpenVINO_Version.at(0) >= 2024 && - global_context_.OpenVINO_Version.at(1) >= 4 )){ + if((global_context_->OpenVINO_Version.at(0) >= 2024 && + global_context_->OpenVINO_Version.at(1) >= 3 )){ std::pair device_property; - device_property = std::make_pair("WORKLOAD_TYPE", global_context_.workload_type); + device_property = std::make_pair("WORKLOAD_TYPE", global_context_->workload_type); device_config.emplace(ov::device::properties("NPU", device_property)); - LOGS_DEFAULT(INFO) << log_tag << "Set workloadtype as " << global_context_.workload_type; + LOGS_DEFAULT(INFO) << log_tag << "Set compile time workloadtype as " << global_context_->workload_type; } } @@ -275,9 +274,9 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } size_t batch_slice_idx = 0; if (subgraph_context_.has_dynamic_input_shape && - !global_context_.disable_dynamic_shapes && - (global_context_.device_type.find("CPU") != std::string::npos || - global_context_.device_type.find("GPU") != std::string::npos)) { + !global_context_->disable_dynamic_shapes && + (global_context_->device_type.find("CPU") != std::string::npos || + global_context_->device_type.find("GPU") != std::string::npos)) { auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); auto tensor_shape = tensor_info.GetShape(); @@ -292,7 +291,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto input = graph_input_info.at(input_idx); OVTensorPtr tensor_ptr; // avoid input copies on the CPU device - if (global_context_.device_type.find("CPU") != std::string::npos) { + if (global_context_->device_type.find("CPU") != std::string::npos) { tensor_ptr = std::make_shared(input.get_element_type(), input_tensor_shape, (void*)tensor_data); } else { @@ -405,10 +404,8 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } // Start Async inference - // exe_network_.Get().set_property(ov::device::properties("NPU", - // {{"WORKLOAD_TYPE", global_context_.runtime_workload_type}})); - std::cout << " Global context runtime workload type = " << global_context_.runtime_workload_type << std::endl; - exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; + exe_network_.Get().set_property(ov::workload_type(global_context_->runtime_workload_type)); infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); @@ -517,9 +514,8 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe } // Start Async inference - // exe_network_.Get().set_property(ov::device::properties("NPU", - // {{"WORKLOAD_TYPE", global_context_.runtime_workload_type}})); - exe_network_.Get().set_property(ov::workload_type(global_context_.runtime_workload_type)); + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; + exe_network_.Get().set_property(ov::workload_type(global_context_->runtime_workload_type)); infer_request->StartAsync(); } catch (const char* msg) { @@ -656,8 +652,8 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { infer_request = inferRequestsQueue_->getIdleRequest(); #ifdef IO_BUFFER_ENABLED - if ((global_context_.device_type.find("GPU") != std::string::npos) && - (global_context_.context != nullptr) && global_context_.is_wholly_supported_graph) { + if ((global_context_->device_type.find("GPU") != std::string::npos) && + (global_context_->context != nullptr) && global_context_->is_wholly_supported_graph) { try { StartRemoteAsyncInference(context, infer_request); } catch (std::string const& msg) { @@ -701,7 +697,7 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { #ifndef IO_BUFFER_ENABLED // Printing performance counts is disabled when IO_BUFFER_ENABLED if (openvino_ep::backend_utils::IsDebugEnabled()) { inferRequestsQueue_->printstatus(); // Printing the elements of infer_requests_ vector pool only in debug mode - std::string& hw_target = global_context_.device_type; + std::string& hw_target = global_context_->device_type; printPerformanceCounts(std::move(infer_request_), std::cout, hw_target); } #endif diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index ea44733974a1c..c61c7cdd84de3 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -30,7 +30,7 @@ class InferRequestsQueue; class BasicBackend : public IBackend { public: BasicBackend(std::unique_ptr& model_proto, - GlobalContext& global_context, + GlobalContext* global_context, const SubGraphContext& subgraph_context, EPCtxHandler& ep_ctx_handle); @@ -56,7 +56,7 @@ class BasicBackend : public IBackend { void CompleteAsyncInference(Ort::KernelContext& context, std::shared_ptr infer_request); - GlobalContext& global_context_; + GlobalContext* global_context_; SubGraphContext subgraph_context_; mutable std::mutex compute_lock_; std::shared_ptr ie_cnn_network_; diff --git a/onnxruntime/core/providers/openvino/ibackend.h b/onnxruntime/core/providers/openvino/ibackend.h index 7a2d6f4e8cd69..d6a836073f3eb 100644 --- a/onnxruntime/core/providers/openvino/ibackend.h +++ b/onnxruntime/core/providers/openvino/ibackend.h @@ -21,7 +21,7 @@ class BackendFactory { public: static std::shared_ptr MakeBackend(std::unique_ptr& model_proto, - GlobalContext& global_context, + GlobalContext* global_context, const SubGraphContext& subgraph_context, EPCtxHandler& ctx_handle); }; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index aad22264ecaed..6c09ed9b1d45b 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -146,7 +146,7 @@ common::Status OpenVINOExecutionProvider::Compile( // For original model, check if the user wants to export a model with pre-compiled blob std::shared_ptr backend_manager = - std::make_shared(*global_context_, + std::make_shared(global_context_.get(), fused_node, graph_body_viewer, *GetLogger(), @@ -184,18 +184,15 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptions& run_options) { - // std::string workload_type=""; auto workload_type_opt = run_options.GetConfigOptions().GetConfigEntry(kOrtRunOptionsWorkloadType); if(workload_type_opt.has_value()){ std::string workload_type = workload_type_opt.value(); - std::cout << " Workload type from RunOption = " << workload_type << std::endl; + LOGS_DEFAULT(INFO) << "[OpenVINO-EP]" << "Workload type from ORT RunOption = " << workload_type; std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); if (workload_type=="default") { global_context_->runtime_workload_type = "DEFAULT"; - // backend_manager_->GetGlobalContext().runtime_workload_type = "DEFAULT"; } else if(workload_type=="efficient") { global_context_->runtime_workload_type = "EFFICIENT"; - // backend_manager_->GetGlobalContext().runtime_workload_type = "EFFICIENT"; } } return Status::OK(); From 76ca087425544d9a7f23be26b4197028c994dbb7 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Fri, 20 Sep 2024 10:43:24 +0530 Subject: [PATCH 10/13] Reset runtime_workload_type on run end --- .../providers/openvino/backends/basic_backend.cc | 14 +++++++------- .../openvino/openvino_execution_provider.cc | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index b2c048381a62b..5629552c39f59 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -175,8 +175,8 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } device_config.emplace(ov::device::properties("NPU", device_property)); #if (OPENVINO_VERSION_MAJOR >= 2024) && (OPENVINO_VERSION_MINOR > 3) - if (global_context_.export_ep_ctx_blob) { - global_context_.ie_core.Get().set_property("NPU", ov::intel_npu::bypass_umd_caching(true)); + if (global_context_->export_ep_ctx_blob) { + global_context_->ie_core.Get().set_property("NPU", ov::intel_npu::bypass_umd_caching(true)); } #endif } @@ -305,8 +305,8 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - if ((global_context_.device_type.find("CPU") != std::string::npos || - global_context_.device_type.find("GPU") != std::string::npos)) { + if ((global_context_->device_type.find("CPU") != std::string::npos || + global_context_->device_type.find("GPU") != std::string::npos)) { OVTensorPtr graph_input_blob; try { graph_input_blob = infer_request->GetTensor(input_name); @@ -351,7 +351,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } input_idx++; } - if (global_context_.device_type.find("NPU") != std::string::npos) { + if (global_context_->device_type.find("NPU") != std::string::npos) { // Set the output blob as remote blob auto graph_output_info = exe_network_.Get().outputs(); auto output_idx = 0; @@ -557,8 +557,8 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe " doesn't exist in the " "list of OpenVINO output tensor names"); } - if ((global_context_.device_type.find("CPU") != std::string::npos || - global_context_.device_type.find("GPU") != std::string::npos)) { + if ((global_context_->device_type.find("CPU") != std::string::npos || + global_context_->device_type.find("GPU") != std::string::npos)) { try { graph_output_blob = infer_request->GetTensor(output_name); } catch (const char* msg) { diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 6c09ed9b1d45b..3442970251463 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -195,11 +195,12 @@ common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptio global_context_->runtime_workload_type = "EFFICIENT"; } } - return Status::OK(); + return Status::OK(); } common::Status OpenVINOExecutionProvider::OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& run_options) { - return Status::OK(); + global_context_->runtime_workload_type = ""; + return Status::OK(); } #ifdef USE_OVEP_NPU_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { From 7e9caabfba0de9aee6233af4b019449963b93080 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Fri, 20 Sep 2024 12:01:32 +0530 Subject: [PATCH 11/13] Fix lint issues --- .../openvino/backends/basic_backend.cc | 37 +++++++++---------- .../openvino/openvino_execution_provider.cc | 9 ++--- .../openvino/openvino_execution_provider.h | 1 + .../openvino/openvino_provider_factory.cc | 4 +- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 5629552c39f59..92a0737f40f77 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -48,7 +48,7 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr // Set the inference_num_threads property of the CPU SetNumThreads(device_config); - //set workload type to decide on the performance mode + // set workload type to decide on the performance mode SetWorkLoadType(device_config); try { @@ -59,8 +59,8 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr if (is_ep_ctx_graph_) { std::istringstream model_stream(ep_ctx_handle.GetModelBlobString()); exe_network_ = global_context_->ie_core.ImportModel(model_stream, - remote_context_, - subgraph_context_.subgraph_name); + remote_context_, + subgraph_context_.subgraph_name); } else if ((global_context_->device_type.find("GPU") != std::string::npos) && (global_context_->context != nullptr)) { LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled"; @@ -80,10 +80,10 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr // If the blob is held in an EPContext node, then skip FE+Compile // and directly move on to creating a backend with the executable blob exe_network_ = global_context_->ie_core.ImportModel(ep_ctx_handle.GetModelBlobStream(), - hw_target, - device_config, - global_context_->ep_context_embed_mode, - subgraph_context_.subgraph_name); + hw_target, + device_config, + global_context_->ep_context_embed_mode, + subgraph_context_.subgraph_name); } else if (global_context_->export_ep_ctx_blob && hw_target.find("NPU") != std::string::npos) { std::shared_ptr ov_model; @@ -102,9 +102,9 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr // Inputs with static dimenstions const std::string model = model_proto->SerializeAsString(); exe_network_ = global_context_->ie_core.CompileModel(model, - hw_target, - device_config, - subgraph_context_.subgraph_name); + hw_target, + device_config, + subgraph_context_.subgraph_name); } else { // For all other types use ov::Model Type ie_cnn_network_ = CreateOVModel(*model_proto, global_context_, const_outputs_map_); exe_network_ = global_context_->ie_core.CompileModel( @@ -235,17 +235,16 @@ void BasicBackend::SetNumThreads(ov::AnyMap& device_config) { device_config.emplace(ov::inference_num_threads(global_context_->num_of_threads)); } -void BasicBackend::SetWorkLoadType(ov::AnyMap& device_config){ - if((global_context_->OpenVINO_Version.at(0) >= 2024 && - global_context_->OpenVINO_Version.at(1) >= 3 )){ - std::pair device_property; - device_property = std::make_pair("WORKLOAD_TYPE", global_context_->workload_type); - device_config.emplace(ov::device::properties("NPU", device_property)); - LOGS_DEFAULT(INFO) << log_tag << "Set compile time workloadtype as " << global_context_->workload_type; - } +void BasicBackend::SetWorkLoadType(ov::AnyMap& device_config) { + if ((global_context_->OpenVINO_Version.at(0) >= 2024 && + global_context_->OpenVINO_Version.at(1) >= 3)) { + std::pair device_property; + device_property = std::make_pair("WORKLOAD_TYPE", global_context_->workload_type); + device_config.emplace(ov::device::properties("NPU", device_property)); + LOGS_DEFAULT(INFO) << log_tag << "Set compile time workloadtype as " << global_context_->workload_type; + } } - // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on // an Infer Request indexed by infer_req_idx void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) { diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 3442970251463..63ee1791736e4 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -185,13 +185,13 @@ common::Status OpenVINOExecutionProvider::Compile( } common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptions& run_options) { auto workload_type_opt = run_options.GetConfigOptions().GetConfigEntry(kOrtRunOptionsWorkloadType); - if(workload_type_opt.has_value()){ + if (workload_type_opt.has_value()) { std::string workload_type = workload_type_opt.value(); LOGS_DEFAULT(INFO) << "[OpenVINO-EP]" << "Workload type from ORT RunOption = " << workload_type; std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); - if (workload_type=="default") { + if (workload_type == "default") { global_context_->runtime_workload_type = "DEFAULT"; - } else if(workload_type=="efficient") { + } else if (workload_type == "efficient") { global_context_->runtime_workload_type = "EFFICIENT"; } } @@ -201,7 +201,7 @@ common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptio common::Status OpenVINOExecutionProvider::OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& run_options) { global_context_->runtime_workload_type = ""; return Status::OK(); - } +} #ifdef USE_OVEP_NPU_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info{ @@ -216,5 +216,4 @@ std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() } #endif - } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index ac2b13503ed2d..51249c190928a 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -198,6 +198,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { std::vector CreatePreferredAllocators() override; #endif std::shared_ptr backend_manager_; + private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 07e56e8213a30..8913476a4fd9d 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -325,9 +325,9 @@ struct OpenVINO_Provider : Provider { if (provider_options_map.find("workload_type") != provider_options_map.end()) { workload_type = provider_options_map.at("workload_type"); std::transform(workload_type.begin(), workload_type.end(), workload_type.begin(), ::tolower); - if (workload_type=="" || workload_type=="default") { + if (workload_type == "" || workload_type == "default") { workload_type = "DEFAULT"; - } else if(workload_type=="efficient") { + } else if (workload_type == "efficient") { workload_type = "EFFICIENT"; } else { ORT_THROW("[ERROR] [OpenVINO] Invalid workload_type - Supported modes are Default and Efficient \n"); From 0f007a3b028916bd632d790b014de66ea7b8e59c Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Fri, 20 Sep 2024 14:20:32 +0530 Subject: [PATCH 12/13] Add check for workload_type --- .../core/providers/openvino/backend_manager.cc | 1 - .../providers/openvino/backends/basic_backend.cc | 16 ++++++++++------ .../openvino/openvino_execution_provider.cc | 4 +--- .../openvino/openvino_execution_provider.h | 1 - 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index 7aa75cba3e87f..71292f5c72ecb 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -31,7 +31,6 @@ BackendManager::BackendManager(GlobalContext* global_context, const logging::Logger& logger, EPCtxHandler& ep_ctx_handle_) { global_context_ = global_context; - openvino_sdk_version_ = std::to_string(global_context_->OpenVINO_Version.at(0)) + "." + std::to_string(global_context_->OpenVINO_Version.at(1)); if (ep_ctx_handle_.CheckForOVEPCtxNode(subgraph, openvino_sdk_version_)) { diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 92a0737f40f77..162118c7089f7 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -403,9 +403,11 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } // Start Async inference - LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; - exe_network_.Get().set_property(ov::workload_type(global_context_->runtime_workload_type)); - infer_request->StartAsync(); + std::string runtime_workload_type = global_context_->runtime_workload_type; + if(runtime_workload_type=="DEFAULT" || runtime_workload_type=="EFFICIENT"){ + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; + exe_network_.Get().set_property(ov::workload_type(runtime_workload_type)); + } } catch (const char* msg) { ORT_THROW(msg); } @@ -513,9 +515,11 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe } // Start Async inference - LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; - exe_network_.Get().set_property(ov::workload_type(global_context_->runtime_workload_type)); - + std::string runtime_workload_type = global_context_->runtime_workload_type; + if(runtime_workload_type=="DEFAULT" || runtime_workload_type=="EFFICIENT"){ + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; + exe_network_.Get().set_property(ov::workload_type(runtime_workload_type)); + } infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 63ee1791736e4..ec44d4a6adf27 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -144,14 +144,12 @@ common::Status OpenVINOExecutionProvider::Compile( // During backend creation, we check if user wants to use precompiled blob onnx model or the original model // For precompiled blob, directly load the model instead of compiling the model // For original model, check if the user wants to export a model with pre-compiled blob - std::shared_ptr backend_manager = std::make_shared(global_context_.get(), fused_node, graph_body_viewer, *GetLogger(), ep_ctx_handle_); - backend_manager_ = backend_manager; compute_info.create_state_func = [backend_manager](ComputeContext* context, FunctionState* state) { OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState(); @@ -199,7 +197,7 @@ common::Status OpenVINOExecutionProvider::OnRunStart(const onnxruntime::RunOptio } common::Status OpenVINOExecutionProvider::OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& run_options) { - global_context_->runtime_workload_type = ""; + global_context_->runtime_workload_type = global_context_->workload_type; return Status::OK(); } #ifdef USE_OVEP_NPU_MEMORY diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 51249c190928a..71fc9b97d7404 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -197,7 +197,6 @@ class OpenVINOExecutionProvider : public IExecutionProvider { #ifdef USE_OVEP_NPU_MEMORY std::vector CreatePreferredAllocators() override; #endif - std::shared_ptr backend_manager_; private: std::unique_ptr global_context_; From a8c4f0b82b81e8bf505c1ef94d13a9607c8b6b37 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Thu, 26 Sep 2024 12:55:53 +0530 Subject: [PATCH 13/13] Fix infer request bug --- onnxruntime/core/providers/openvino/backends/basic_backend.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 162118c7089f7..bf7aa60d43dcc 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -408,6 +408,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << global_context_->runtime_workload_type << " mode is set for OV inference"; exe_network_.Get().set_property(ov::workload_type(runtime_workload_type)); } + infer_request->StartAsync(); } catch (const char* msg) { ORT_THROW(msg); }