From 89094a812bdd26ecf2323243b538e454c578a219 Mon Sep 17 00:00:00 2001 From: "Javier E. Martinez" Date: Thu, 18 Jul 2024 16:00:28 -0700 Subject: [PATCH 01/23] Prototype shared memory allocator on Windows using OV-EP --- .../onnxruntime/core/framework/allocator.h | 2 + .../openvino/backends/basic_backend.cc | 164 ++++++++++++++++-- .../openvino/backends/basic_backend.h | 9 + 3 files changed, 161 insertions(+), 14 deletions(-) diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 097873c5e3653..377ef6157d693 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -50,6 +50,8 @@ constexpr const char* HIP = "Hip"; constexpr const char* HIP_PINNED = "HipPinned"; constexpr const char* OpenVINO_CPU = "OpenVINO_CPU"; constexpr const char* OpenVINO_GPU = "OpenVINO_GPU"; +constexpr const char* OpenVINO_RT = "OpenVINO_RT"; +constexpr const char* WIN32_HANDLE = "WIN32_HANDLE"; constexpr const char* WEBGPU_BUFFER = "WebGPU_Buffer"; constexpr size_t kAllocAlignment = 256; diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d79aa35be6418..11ac474aac407 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -14,6 +14,11 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/backend_manager.h" +#include +#include + +#pragma comment(lib, "opencl") + namespace onnxruntime { namespace openvino_ep { @@ -287,16 +292,144 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - OVTensorPtr graph_input_blob; + auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_key; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_key = it->second; + } else { + if (allocator_name == OpenVINO_RT) { + auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); + auto tensor_shape = tensor_info.GetShape(); + auto tensor_size = tensor_shape.size(); + ov::Shape input_tensor_shape = ov::Shape(tensor_size, 0); + const char* tensor_data = tensor.GetTensorData(); + std::map ov_type_convert{ + {ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, ov::element::f32}}; + auto ov_type = ov_type_convert[tensor_info.GetElementType()]; + ov_tensor_key.tensor_ptr = std::make_shared(ov_type, input_tensor_shape, + (void*)tensor_data); + ov_tensor_key.copy_needed = false; + } else if (allocator_name == WIN32_HANDLE) { + const void* tensor_data = tensor.GetTensorRawData(); + auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); + cl_context ctx = remote_context_.get(); + + cl_mem_properties extMemProperties[] = {CL_EXTERNAL_MEMORY_HANDLE_OPAQUE_WIN32_KHR, (cl_mem_properties)tensor_data, + 0}; + cl_int errcode = 0; + auto shape = input_info_iter->get_shape(); + size_t elem_count = std::reduce( + shape.begin()++, shape.end(), *shape.begin(), std::multiplies<>()); + auto elem_size = input_info_iter->get_element_type().bitwidth(); + cl_mem extMemBuffer = + clCreateBufferWithProperties(remote_context_, extMemProperties, 0, elem_count * elem_size, nullptr, &errcode); + auto remote_tensor = + remote_context_.create_tensor(input_info_iter->get_element_type(), input_info_iter->get_shape(), extMemBuffer); + ov_tensor_key.tensor_ptr = std::make_shared(remote_tensor); + ov_tensor_key.copy_needed = false; + + } else { + auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); + auto remote_tensor = remote_context_.create_host_tensor(input_info_iter->get_element_type(), input_info_iter->get_shape()); + ov_tensor_key.tensor_ptr = std::make_shared(remote_tensor); + ov_tensor_key.copy_needed = true; + } + + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); + } + + if (ov_tensor_key.copy_needed) { + const char* ort_tensor_data = tensor.GetTensorData(); + size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; + std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); + } + try { - graph_input_blob = infer_request->GetTensor(input_name); + infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); } catch (const char* msg) { ORT_THROW(msg); } - FillInputBlob(std::move(graph_input_blob), batch_slice_idx, std::move(input_name), context, subgraph_context_); } input_idx++; } + + // Set the output blob as remote blob + auto graph_output_info = exe_network_.Get().outputs(); + for (auto output_info_iter = graph_output_info.begin(); + output_info_iter != graph_output_info.end(); ++output_info_iter) { + auto output_names = output_info_iter->get_names(); + std::string onnx_output_name; + std::string output_name; + bool output_name_found = false; + // using the output name retrieved from ONNX original to match with the output names returned by OV tensors + for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { + onnx_output_name = it->first; + if (output_names.find(onnx_output_name) != output_names.end()) { + // Assigning the output_name + output_name = it->first; + output_name_found = true; + break; + } + } + if (!output_name_found) { + ORT_THROW( + log_tag + + "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + + onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names"); + } + + size_t batch_size = 1; + Ort::UnownedValue tensor = GetOutputTensor(context, + batch_size, + infer_request, + output_name, + subgraph_context_.output_names); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; + } else { + // Check if ORT Value wraps a device pointer + if (allocator_name == WIN32_HANDLE) { + const void* tensor_data = tensor.GetTensorRawData(); + auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); + cl_context ctx = remote_context_.get(); + + cl_mem_properties extMemProperties[] = {CL_EXTERNAL_MEMORY_HANDLE_OPAQUE_WIN32_KHR, (cl_mem_properties)tensor_data, + 0}; + cl_int errcode = 0; + auto shape = output_info_iter->get_shape(); + size_t elem_count = std::reduce( + shape.begin()++, shape.end(), *shape.begin(), std::multiplies<>()); + auto elem_size = output_info_iter->get_element_type().bitwidth(); + cl_mem extMemBuffer = + clCreateBufferWithProperties(remote_context_, extMemProperties, 0, elem_count * elem_size, nullptr, &errcode); + auto remote_tensor = + remote_context_.create_tensor(output_info_iter->get_element_type(), output_info_iter->get_shape(), extMemBuffer); + ov::Tensor tensor_t = static_cast(remote_tensor); + ov_tensor_data.tensor_ptr = std::make_shared(tensor_t); + ov_tensor_data.copy_needed = false; + } else if (allocator_name == CPU) { + auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); + auto remote_tensor = remote_context_.create_host_tensor(output_info_iter->get_element_type(), output_info_iter->get_shape()); + ov_tensor_data.tensor_ptr = std::make_shared(remote_tensor); + ov_tensor_data.copy_needed = true; + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); + } + + try { + infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } + } + // Start Async inference infer_request->StartAsync(); } catch (const char* msg) { @@ -422,7 +555,6 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe auto graph_output_info = exe_network_.Get().outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { - OVTensorPtr graph_output_blob; auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; @@ -446,20 +578,24 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe " doesn't exist in the " "list of OpenVINO output tensor names"); } - try { - graph_output_blob = infer_request->GetTensor(output_name); - } catch (const char* msg) { - ORT_THROW(msg); - } + size_t batch_size = 1; Ort::UnownedValue output_tensor = GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); - auto mem_info = output_tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - return; + auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; } else { - size_t batch_slice = 0; - FillOutputBlob(std::move(graph_output_blob), output_tensor, batch_slice); + ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); + } + + if (ov_tensor_data.copy_needed) { + auto ort_tensor_data = output_tensor.GetTensorMutableData(); + size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; + std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); } } diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index bcd3161590ba0..a83acf6e4938e 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/openvino/contexts.h" @@ -20,6 +21,11 @@ namespace onnxruntime { namespace openvino_ep { +struct ov_tensor_data_t { + OVTensorPtr tensor_ptr; + bool copy_needed; +}; + class InferRequestsQueue; class BasicBackend : public IBackend { public: @@ -60,6 +66,9 @@ class BasicBackend : public IBackend { #if defined IO_BUFFER_ENABLED OVRemoteContextPtr remote_context_; #endif + + using ort_tensor_key_t = std::pair; + std::map ort_ov_tensor_map; }; class InferRequestsQueue { From 2e4b2053ac1998d934605ca71c01a2960075a254 Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Fri, 23 Aug 2024 10:47:41 -0700 Subject: [PATCH 02/23] Partially working allocator. Crashing on tensor destruction. Might have UMD exceptions. Needs further debug. Unknown if values are correct. --- .../onnxruntime/core/framework/allocator.h | 7 ++ onnxruntime/core/framework/allocator.cc | 4 + .../openvino/backends/basic_backend.cc | 84 +++++-------------- .../openvino/openvino_execution_provider.cc | 13 +++ .../openvino/openvino_execution_provider.h | 2 + .../core/providers/openvino/ov_allocator.cc | 44 ++++++++++ .../core/providers/openvino/ov_allocator.h | 23 +++++ 7 files changed, 115 insertions(+), 62 deletions(-) create mode 100644 onnxruntime/core/providers/openvino/ov_allocator.cc create mode 100644 onnxruntime/core/providers/openvino/ov_allocator.h diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 377ef6157d693..ea92cee26a25e 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -50,7 +50,14 @@ constexpr const char* HIP = "Hip"; constexpr const char* HIP_PINNED = "HipPinned"; constexpr const char* OpenVINO_CPU = "OpenVINO_CPU"; constexpr const char* OpenVINO_GPU = "OpenVINO_GPU"; +constexpr const char* OpenVINO_NPU = "OpenVINO_RT_NPU"; + +// application +// 1. Allocate with ORT::CreateTensor("") +// 2. "Manual" allocation + constexpr const char* OpenVINO_RT = "OpenVINO_RT"; +constexpr const char* OpenVINO_RT_NPU = "OpenVINO_RT_NPU"; constexpr const char* WIN32_HANDLE = "WIN32_HANDLE"; constexpr const char* WEBGPU_BUFFER = "WebGPU_Buffer"; diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc index c3e96e450c59b..5e66f2b99fded 100644 --- a/onnxruntime/core/framework/allocator.cc +++ b/onnxruntime/core/framework/allocator.cc @@ -145,6 +145,10 @@ ORT_API_STATUS_IMPL(OrtApis::CreateMemoryInfo, _In_ const char* name1, enum OrtA *out = new OrtMemoryInfo( name1, type, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, static_cast(id1)), id1, mem_type1); + } else if (strcmp(name1, onnxruntime::OpenVINO_RT_NPU) == 0) { + *out = new OrtMemoryInfo( + name1, type, OrtDevice(OrtDevice::NPU, OrtDevice::MemType::DEFAULT, static_cast(id1)), id1, + mem_type1); } else if (strcmp(name1, onnxruntime::CUDA_PINNED) == 0) { *out = new OrtMemoryInfo( onnxruntime::CUDA_PINNED, type, OrtDevice(OrtDevice::CPU, OrtDevice::MemType::CUDA_PINNED, static_cast(id1)), diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 11ac474aac407..2f2ed5e5e5f05 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -14,11 +14,6 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/backend_manager.h" -#include -#include - -#pragma comment(lib, "opencl") - namespace onnxruntime { namespace openvino_ep { @@ -259,6 +254,12 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); } + auto ort_shape_to_ovshape = [](const std::vector& shape) { + ov::Shape ov_shape(shape.size()); + std::copy(shape.begin(), shape.end(), ov_shape.begin()); + return ov_shape; + }; + size_t batch_slice_idx = 0; if (subgraph_context_.has_dynamic_input_shape && !global_context_.disable_dynamic_shapes && @@ -299,44 +300,19 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { ov_tensor_key = it->second; } else { - if (allocator_name == OpenVINO_RT) { - auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); - auto tensor_shape = tensor_info.GetShape(); - auto tensor_size = tensor_shape.size(); - ov::Shape input_tensor_shape = ov::Shape(tensor_size, 0); - const char* tensor_data = tensor.GetTensorData(); - std::map ov_type_convert{ - {ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, ov::element::f32}}; - auto ov_type = ov_type_convert[tensor_info.GetElementType()]; - ov_tensor_key.tensor_ptr = std::make_shared(ov_type, input_tensor_shape, - (void*)tensor_data); - ov_tensor_key.copy_needed = false; - } else if (allocator_name == WIN32_HANDLE) { - const void* tensor_data = tensor.GetTensorRawData(); - auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); - cl_context ctx = remote_context_.get(); - - cl_mem_properties extMemProperties[] = {CL_EXTERNAL_MEMORY_HANDLE_OPAQUE_WIN32_KHR, (cl_mem_properties)tensor_data, - 0}; - cl_int errcode = 0; - auto shape = input_info_iter->get_shape(); - size_t elem_count = std::reduce( - shape.begin()++, shape.end(), *shape.begin(), std::multiplies<>()); - auto elem_size = input_info_iter->get_element_type().bitwidth(); - cl_mem extMemBuffer = - clCreateBufferWithProperties(remote_context_, extMemProperties, 0, elem_count * elem_size, nullptr, &errcode); - auto remote_tensor = - remote_context_.create_tensor(input_info_iter->get_element_type(), input_info_iter->get_shape(), extMemBuffer); - ov_tensor_key.tensor_ptr = std::make_shared(remote_tensor); + // Does this make sense for both types of allocators? + auto input = ie_cnn_network_->get_parameters().at(input_idx); + ov_tensor_key.tensor_ptr = std::make_shared(input->get_element_type(), input->get_shape(), + (void*)tensor.GetTensorRawData()); + if (allocator_name == OpenVINO_RT_NPU) { + // do we need this?? + // auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); + // auto tensor_shape = tensor_info.GetShape(); + // auto tensor_size = tensor_shape.size(); ov_tensor_key.copy_needed = false; - } else { - auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); - auto remote_tensor = remote_context_.create_host_tensor(input_info_iter->get_element_type(), input_info_iter->get_shape()); - ov_tensor_key.tensor_ptr = std::make_shared(remote_tensor); ov_tensor_key.copy_needed = true; } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); } @@ -358,6 +334,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque // Set the output blob as remote blob auto graph_output_info = exe_network_.Get().outputs(); + auto output_idx = 0; for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { auto output_names = output_info_iter->get_names(); @@ -394,30 +371,12 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { ov_tensor_data = it->second; } else { - // Check if ORT Value wraps a device pointer - if (allocator_name == WIN32_HANDLE) { - const void* tensor_data = tensor.GetTensorRawData(); - auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); - cl_context ctx = remote_context_.get(); - - cl_mem_properties extMemProperties[] = {CL_EXTERNAL_MEMORY_HANDLE_OPAQUE_WIN32_KHR, (cl_mem_properties)tensor_data, - 0}; - cl_int errcode = 0; - auto shape = output_info_iter->get_shape(); - size_t elem_count = std::reduce( - shape.begin()++, shape.end(), *shape.begin(), std::multiplies<>()); - auto elem_size = output_info_iter->get_element_type().bitwidth(); - cl_mem extMemBuffer = - clCreateBufferWithProperties(remote_context_, extMemProperties, 0, elem_count * elem_size, nullptr, &errcode); - auto remote_tensor = - remote_context_.create_tensor(output_info_iter->get_element_type(), output_info_iter->get_shape(), extMemBuffer); - ov::Tensor tensor_t = static_cast(remote_tensor); - ov_tensor_data.tensor_ptr = std::make_shared(tensor_t); + auto output = ie_cnn_network_->get_results().at(output_idx); + ov_tensor_data.tensor_ptr = std::make_shared(output->get_element_type(), output->get_shape(), + (void*)tensor.GetTensorRawData()); + if(allocator_name == OpenVINO_RT_NPU) { ov_tensor_data.copy_needed = false; - } else if (allocator_name == CPU) { - auto remote_context_ = global_context_.ie_core.Get().get_default_context("GPU").as(); - auto remote_tensor = remote_context_.create_host_tensor(output_info_iter->get_element_type(), output_info_iter->get_shape()); - ov_tensor_data.tensor_ptr = std::make_shared(remote_tensor); + } else { ov_tensor_data.copy_needed = true; } ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); @@ -428,6 +387,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } catch (const char* msg) { ORT_THROW(msg); } + output_idx++; } // Start Async inference diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 29c45916795d3..2f7b4e506c1b8 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,6 +10,7 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" +#include "core/providers/openvino/ov_allocator.h" #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) @@ -180,4 +181,16 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } +std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { + AllocatorCreationInfo npu_allocator_info { + [](OrtDevice::DeviceId device_id) { + return std::make_unique(OrtDevice::NPU, device_id, OpenVINO_RT_NPU); + }, + 0, + }; + + // fill in allocator + return std::vector{CreateAllocator(npu_allocator_info)}; +} + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 030e5bba71b67..5f8f048ccd4ad 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -190,6 +190,8 @@ class OpenVINOExecutionProvider : public IExecutionProvider { return nullptr; } + std::vector CreatePreferredAllocators() override; + private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc new file mode 100644 index 0000000000000..41ee7cf0d5112 --- /dev/null +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -0,0 +1,44 @@ +// Copyright (C) Intel Corporation +// Licensed under the MIT License + +#include "core/providers/openvino/ov_allocator.h" +#include "core/providers/openvino/ov_interface.h" +#include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +namespace onnxruntime { + +using namespace openvino_ep; + +OVRTAllocator::OVRTAllocator(OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name) : IAllocator(OrtMemoryInfo(name, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(device_type, OrtDevice::MemType::DEFAULT, device_id), device_id, OrtMemTypeCPUInput)) { + if(device_type == OrtDevice::NPU) { + ov::Core core; + remote_ctx_ = core.get_default_context("NPU").as(); + } else { + ORT_THROW("Invalid device type"); + } +} + +void* OVRTAllocator::Alloc(size_t size) { + try { + // TODO: probably want to handle alignment + ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, {size + sizeof(ov::Tensor*)})); + ov::Tensor** ptr = reinterpret_cast(tensor->data()); + *ptr = tensor; + return reinterpret_cast (ptr + 1); + } catch (const ov::Exception& e) { + ORT_THROW(std::string("Alloc failed: ") + e.what()); + } + return nullptr; +} + +void OVRTAllocator::Free(void* p) { + try { + ov::Tensor **ptr = reinterpret_cast(p); + delete ptr[-1]; + } catch (const ov::Exception& e) { + ORT_THROW(std::string("Free failed: ") + e.what()); + } + } + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h new file mode 100644 index 0000000000000..c220c5c770888 --- /dev/null +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -0,0 +1,23 @@ +// Copyright (C) Intel Corporation +// Licensed under the MIT License + +#pragma once + +#include "core/common/inlined_containers.h" +#include "core/framework/allocator.h" +#include "openvino/runtime/remote_context.hpp" + + +namespace onnxruntime { + +class OVRTAllocator : public IAllocator { + public: + OVRTAllocator(OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); + void* Alloc(size_t size) override; + void Free(void* p) override; + + private: + ov::RemoteContext remote_ctx_; +}; + +} // namespace onnxruntime From 63e8aee5761424e6133fee9b7750e15f9aa175cf Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Fri, 23 Aug 2024 17:12:25 -0700 Subject: [PATCH 03/23] Hard code onnx perf to use RT NPU allocator for inputs --- onnxruntime/test/perftest/ort_test_session.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index fc1bdb10d7453..4511407ebe5d2 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -940,7 +940,7 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { // iterate over all input nodes for (size_t i = 0; i < static_cast(input_length_); i++) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUInput); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); std::vector input_node_dim = tensor_info.GetShape(); @@ -952,7 +952,7 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { } } - auto allocator = Ort::AllocatorWithDefaultOptions(); + Ort::Allocator allocator(session_, memory_info); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), input_node_dim.size(), tensor_info.GetElementType()); InitializeTensorWithSeed(seed, input_tensor); From cd88b0ce9fb63de27d302b4007cfa13f7f6b3eea Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 26 Aug 2024 15:03:49 -0700 Subject: [PATCH 04/23] Fix allocation lookups coming from different level zero contexts --- .../core/providers/openvino/openvino_execution_provider.cc | 4 ++-- onnxruntime/core/providers/openvino/ov_allocator.cc | 5 ++--- onnxruntime/core/providers/openvino/ov_allocator.h | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 2f7b4e506c1b8..534b05a352310 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -183,8 +183,8 @@ common::Status OpenVINOExecutionProvider::Compile( std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info { - [](OrtDevice::DeviceId device_id) { - return std::make_unique(OrtDevice::NPU, device_id, OpenVINO_RT_NPU); + [this](OrtDevice::DeviceId device_id) { + return std::make_unique(global_context_->ie_core.Get(), OrtDevice::NPU, device_id, OpenVINO_RT_NPU); }, 0, }; diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 41ee7cf0d5112..2473f32a2baa0 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -10,10 +10,9 @@ namespace onnxruntime { using namespace openvino_ep; -OVRTAllocator::OVRTAllocator(OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name) : IAllocator(OrtMemoryInfo(name, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(device_type, OrtDevice::MemType::DEFAULT, device_id), device_id, OrtMemTypeCPUInput)) { +OVRTAllocator::OVRTAllocator(ov::Core &core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name) : IAllocator(OrtMemoryInfo(name, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(device_type, OrtDevice::MemType::DEFAULT, device_id), device_id, OrtMemTypeCPUInput)), core_(core) { if(device_type == OrtDevice::NPU) { - ov::Core core; - remote_ctx_ = core.get_default_context("NPU").as(); + remote_ctx_ = core_.get_default_context("NPU").as(); } else { ORT_THROW("Invalid device type"); } diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index c220c5c770888..46a3afdf2274b 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -12,11 +12,12 @@ namespace onnxruntime { class OVRTAllocator : public IAllocator { public: - OVRTAllocator(OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); + OVRTAllocator(ov::Core &core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); void* Alloc(size_t size) override; void Free(void* p) override; private: + ov::Core &core_; ov::RemoteContext remote_ctx_; }; From 89127f0ab1e61653228d1602a01279717af2d273 Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 26 Aug 2024 15:44:09 -0700 Subject: [PATCH 05/23] Page align OV allocation --- .../core/providers/openvino/ov_allocator.cc | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 2473f32a2baa0..6a6939b74202e 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -10,21 +10,32 @@ namespace onnxruntime { using namespace openvino_ep; -OVRTAllocator::OVRTAllocator(ov::Core &core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name) : IAllocator(OrtMemoryInfo(name, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(device_type, OrtDevice::MemType::DEFAULT, device_id), device_id, OrtMemTypeCPUInput)), core_(core) { - if(device_type == OrtDevice::NPU) { - remote_ctx_ = core_.get_default_context("NPU").as(); - } else { - ORT_THROW("Invalid device type"); - } +constexpr size_t default_alignment = 4096; + +static inline size_t align_up(size_t size, size_t pow2_alignment) { + return (size + pow2_alignment - 1) & ~(pow2_alignment - 1); +} + +OVRTAllocator::OVRTAllocator(ov::Core& core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name) : IAllocator(OrtMemoryInfo(name, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(device_type, OrtDevice::MemType::DEFAULT, device_id), device_id, OrtMemTypeCPUInput)), core_(core) { + if (device_type == OrtDevice::NPU) { + remote_ctx_ = core_.get_default_context("NPU").as(); + } else { + ORT_THROW("Invalid device type"); + } } void* OVRTAllocator::Alloc(size_t size) { try { - // TODO: probably want to handle alignment - ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, {size + sizeof(ov::Tensor*)})); - ov::Tensor** ptr = reinterpret_cast(tensor->data()); - *ptr = tensor; - return reinterpret_cast (ptr + 1); + size_t alloc_size = align_up(size + sizeof(ov::Tensor*) + default_alignment, default_alignment); + ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, + { alloc_size })); + uintptr_t data_ptr = reinterpret_cast(tensor->data()); + + ov::Tensor** ptr = reinterpret_cast(align_up(data_ptr + sizeof(ov::Tensor*), default_alignment)); + ptr[-1] = tensor; + + return reinterpret_cast(ptr); + } catch (const ov::Exception& e) { ORT_THROW(std::string("Alloc failed: ") + e.what()); } @@ -33,11 +44,11 @@ void* OVRTAllocator::Alloc(size_t size) { void OVRTAllocator::Free(void* p) { try { - ov::Tensor **ptr = reinterpret_cast(p); + ov::Tensor** ptr = reinterpret_cast(p); delete ptr[-1]; } catch (const ov::Exception& e) { ORT_THROW(std::string("Free failed: ") + e.what()); - } } +} } // namespace onnxruntime From d43219f9b794d3ff86dd71df162932a44ab4ff42 Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 26 Aug 2024 15:45:29 -0700 Subject: [PATCH 06/23] Allocate input as WC --- onnxruntime/core/providers/openvino/ov_allocator.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 6a6939b74202e..453443dc60d32 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -27,9 +27,10 @@ OVRTAllocator::OVRTAllocator(ov::Core& core, OrtDevice::DeviceType device_type, void* OVRTAllocator::Alloc(size_t size) { try { size_t alloc_size = align_up(size + sizeof(ov::Tensor*) + default_alignment, default_alignment); - ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, - { alloc_size })); - uintptr_t data_ptr = reinterpret_cast(tensor->data()); + + ov::intel_npu::level_zero::ZeroContext npu_ctx = remote_ctx_.as(); + auto tensor = new ov::intel_npu::level_zero::ZeroBufferTensor(npu_ctx.create_l0_host_tensor(ov::element::Type_t::u8, {alloc_size}, ov::intel_npu::TensorType::INPUT)); + uintptr_t data_ptr = reinterpret_cast(tensor->get()); ov::Tensor** ptr = reinterpret_cast(align_up(data_ptr + sizeof(ov::Tensor*), default_alignment)); ptr[-1] = tensor; From 274e6afb93d901a911eccf4d641c03472d63f2af Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 26 Aug 2024 15:46:07 -0700 Subject: [PATCH 07/23] Only set tensors when they have changed. --- .../openvino/backends/basic_backend.cc | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 2f2ed5e5e5f05..04a61dd641950 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -314,6 +314,12 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_key.copy_needed = true; } ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); + + try { + infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } } if (ov_tensor_key.copy_needed) { @@ -322,12 +328,6 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); } - - try { - infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); - } catch (const char* msg) { - ORT_THROW(msg); - } } input_idx++; } @@ -380,13 +380,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_data.copy_needed = true; } ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); - } - try { - infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); - } catch (const char* msg) { - ORT_THROW(msg); + try { + infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } } + output_idx++; } From 6feae8442028cc888e468d7a323b9f781b54c05e Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 26 Aug 2024 16:01:45 -0700 Subject: [PATCH 08/23] Revert "Allocate input as WC" This reverts commit d43219f9b794d3ff86dd71df162932a44ab4ff42. --- onnxruntime/core/providers/openvino/ov_allocator.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 453443dc60d32..6a6939b74202e 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -27,10 +27,9 @@ OVRTAllocator::OVRTAllocator(ov::Core& core, OrtDevice::DeviceType device_type, void* OVRTAllocator::Alloc(size_t size) { try { size_t alloc_size = align_up(size + sizeof(ov::Tensor*) + default_alignment, default_alignment); - - ov::intel_npu::level_zero::ZeroContext npu_ctx = remote_ctx_.as(); - auto tensor = new ov::intel_npu::level_zero::ZeroBufferTensor(npu_ctx.create_l0_host_tensor(ov::element::Type_t::u8, {alloc_size}, ov::intel_npu::TensorType::INPUT)); - uintptr_t data_ptr = reinterpret_cast(tensor->get()); + ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, + { alloc_size })); + uintptr_t data_ptr = reinterpret_cast(tensor->data()); ov::Tensor** ptr = reinterpret_cast(align_up(data_ptr + sizeof(ov::Tensor*), default_alignment)); ptr[-1] = tensor; From c1f3b3ec7caee7024b0ef887fa762d36001e0daa Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Mon, 26 Aug 2024 17:19:40 -0700 Subject: [PATCH 09/23] Hard code onnx perf to use RT NPU for outputs --- onnxruntime/test/perftest/ort_test_session.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 4511407ebe5d2..a222518f13dc4 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -838,7 +838,10 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } size_t output_count = session_.GetOutputCount(); output_names_.resize(output_count); - Ort::AllocatorWithDefaultOptions a; + + Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + Ort::Allocator a(session_, memory_info); + for (size_t i = 0; i != output_count; ++i) { auto output_name = session_.GetOutputNameAllocated(i, a); assert(output_name != nullptr); From 1e3dadd355c1bcd665e83d61a6dc8addc5399ccd Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Tue, 27 Aug 2024 10:26:28 -0700 Subject: [PATCH 10/23] Revert "Hard code onnx perf to use RT NPU for outputs" This reverts commit c1f3b3ec7caee7024b0ef887fa762d36001e0daa. --- onnxruntime/test/perftest/ort_test_session.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index a222518f13dc4..4511407ebe5d2 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -838,10 +838,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } size_t output_count = session_.GetOutputCount(); output_names_.resize(output_count); - - Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); - Ort::Allocator a(session_, memory_info); - + Ort::AllocatorWithDefaultOptions a; for (size_t i = 0; i != output_count; ++i) { auto output_name = session_.GetOutputNameAllocated(i, a); assert(output_name != nullptr); From 61a2d4acda31567e17bd78cdfa5b48d5ee8eeb34 Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Tue, 27 Aug 2024 14:23:48 -0700 Subject: [PATCH 11/23] Hard code onnx perf to use RT NPU for outputs fixed --- onnxruntime/test/perftest/ort_test_session.cc | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 4511407ebe5d2..7126f0f1a96f8 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -34,10 +34,34 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { // Randomly pick one OrtValueArray from test_inputs_. (NOT ThreadSafe) const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); const size_t id = static_cast(dist_(rand_engine_, p)); + + + std::vector outputs; + Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + Ort::Allocator allocator(session_, memory_info); + for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { + Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + + std::vector output_shape = tensor_info.GetShape(); + + // free dimensions are treated as 1 if not overridden + for (int64_t& dim : output_shape) { + if (dim == -1) { + dim = 1; + } + } + + outputs.push_back(Ort::Value::CreateTensor(allocator, (const int64_t*)output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); + } + auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); - auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), output_names_raw_ptr.size()); + + session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + output_names_raw_ptr.data(), outputs.data(), output_names_raw_ptr.size()); + auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration duration_seconds = end - start; return duration_seconds; From 5800966e411739d84b1eca1cec77819f06d819c3 Mon Sep 17 00:00:00 2001 From: Eric Crawford Date: Tue, 27 Aug 2024 15:53:11 -0700 Subject: [PATCH 12/23] Fix onnx_perf_test app crash on tensor destroy --- onnxruntime/test/perftest/ort_test_session.cc | 47 +++++++++---------- onnxruntime/test/perftest/ort_test_session.h | 2 + 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 7126f0f1a96f8..d87eab7ee0754 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -35,32 +35,11 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); const size_t id = static_cast(dist_(rand_engine_, p)); - - std::vector outputs; - Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); - Ort::Allocator allocator(session_, memory_info); - for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { - Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - - std::vector output_shape = tensor_info.GetShape(); - - // free dimensions are treated as 1 if not overridden - for (int64_t& dim : output_shape) { - if (dim == -1) { - dim = 1; - } - } - - outputs.push_back(Ort::Value::CreateTensor(allocator, (const int64_t*)output_shape.data(), - output_shape.size(), tensor_info.GetElementType())); - } - auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), outputs.data(), output_names_raw_ptr.size()); + output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration duration_seconds = end - start; @@ -878,6 +857,25 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); input_names_str_[i] = m.GetInputName(i); input_names_[i] = input_names_str_[i].c_str(); } + + Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + custom_allocator_ = std::make_unique(session_, memory_info); + for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { + Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + + std::vector output_shape = tensor_info.GetShape(); + + // free dimensions are treated as 1 if not overridden + for (int64_t& dim : output_shape) { + if (dim == -1) { + dim = 1; + } + } + + outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); + } } template @@ -964,7 +962,6 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { // iterate over all input nodes for (size_t i = 0; i < static_cast(input_length_); i++) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); - Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUInput); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); std::vector input_node_dim = tensor_info.GetShape(); @@ -976,8 +973,8 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { } } - Ort::Allocator allocator(session_, memory_info); - Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), + + Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), input_node_dim.size(), tensor_info.GetElementType()); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index f1a4220ab325e..729af249ad7a5 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -38,6 +38,8 @@ class OnnxRuntimeTestSession : public TestSession { std::mt19937 rand_engine_; std::uniform_int_distribution dist_; std::vector> test_inputs_; + std::unique_ptr custom_allocator_; + std::vector outputs_; std::vector output_names_; // The same size with output_names_. // TODO: implement a customized allocator, then we can remove output_names_ to simplify this code From 0faaf9f1e55a9511230a735e5092a8badaf7bc80 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Wed, 28 Aug 2024 03:48:10 -0700 Subject: [PATCH 13/23] refactor: remove redundant ort_shape_to_ovshape lambda function --- .../core/providers/openvino/backends/basic_backend.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 446805919b41c..7af570dacff1e 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -262,12 +262,6 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); } - auto ort_shape_to_ovshape = [](const std::vector& shape) { - ov::Shape ov_shape(shape.size()); - std::copy(shape.begin(), shape.end(), ov_shape.begin()); - return ov_shape; - }; - size_t batch_slice_idx = 0; if (subgraph_context_.has_dynamic_input_shape && !global_context_.disable_dynamic_shapes && From a1f92cd4bde43bd24e6ba3c3a1b38521934c0937 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Thu, 29 Aug 2024 01:08:23 -0700 Subject: [PATCH 14/23] alocate buffer in NPU visible region from perf test application --- onnxruntime/test/perftest/ort_test_session.cc | 26 ++++++++++++++----- onnxruntime/test/perftest/ort_test_session.h | 1 + 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index d87eab7ee0754..cc1cf5d2f315d 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -38,8 +38,13 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); + if (!use_device_mem) { + auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + output_names_raw_ptr.data(), output_names_raw_ptr.size()); + } else { session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); + } auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration duration_seconds = end - start; @@ -964,6 +969,9 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + if (!use_device_mem){ + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + } std::vector input_node_dim = tensor_info.GetShape(); // free dimensions are treated as 1 if not overridden @@ -972,12 +980,18 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { dim = 1; } } - - - Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(0, i, std::move(input_tensor)); + if (use_device_mem){ + Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(0, i, std::move(input_tensor)); + } else { + auto allocator = Ort::AllocatorWithDefaultOptions(); + Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(0, i, std::move(input_tensor)); + } } } return true; diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index 729af249ad7a5..e33041a2a0958 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -48,6 +48,7 @@ class OnnxRuntimeTestSession : public TestSession { std::vector input_names_str_; const int input_length_; std::string provider_name_; + bool use_device_mem = false; }; } // namespace perftest From 6ee25da8a0a89cd20ea1b30c4e57d28404150498 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Thu, 29 Aug 2024 02:51:29 -0700 Subject: [PATCH 15/23] remove redundant code --- .../providers/openvino/backends/basic_backend.cc | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 7af570dacff1e..15ded4965bcc1 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -48,14 +48,6 @@ BasicBackend::BasicBackend(std::unique_ptr& model_pr // Set the inference_num_threads property of the CPU SetNumThreads(device_config); -#ifndef NDEBUG - if (IsDebugEnabled()) { - std::string file_name = subgraph_context.subgraph_name + "_static.onnx"; - std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary); - model_proto.SerializeToOstream(outfile); - } -#endif - try { std::string dev_prec = global_context.device_type + "_" + global_context_.precision_str; @@ -307,10 +299,6 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_key.tensor_ptr = std::make_shared(input->get_element_type(), input->get_shape(), (void*)tensor.GetTensorRawData()); if (allocator_name == OpenVINO_RT_NPU) { - // do we need this?? - // auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); - // auto tensor_shape = tensor_info.GetShape(); - // auto tensor_size = tensor_shape.size(); ov_tensor_key.copy_needed = false; } else { ov_tensor_key.copy_needed = true; @@ -389,7 +377,6 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } - output_idx++; } From a9f2357b43f7a704fe32bc23ed104ca9b8e1b970 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Thu, 29 Aug 2024 03:36:22 -0700 Subject: [PATCH 16/23] add command line parameter in perf test for using remote tensors --- onnxruntime/test/perftest/ort_test_session.cc | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index cc1cf5d2f315d..e560da64af73e 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -819,7 +819,12 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " "should be a boolean i.e. true or false. Default value is false.\n"); } - } else { + } else if (key == "use_device_mem") { + if (value == "true" || value == "True") { + use_device_mem = true; + } + } + else { ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } @@ -863,23 +868,25 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); input_names_[i] = input_names_str_[i].c_str(); } - Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); - custom_allocator_ = std::make_unique(session_, memory_info); - for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { - Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + if (use_device_mem) { + Ort::MemoryInfo memory_info = Ort::MemoryInfo("OpenVINO_RT_NPU", OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + custom_allocator_ = std::make_unique(session_, memory_info); + for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { + Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - std::vector output_shape = tensor_info.GetShape(); + std::vector output_shape = tensor_info.GetShape(); - // free dimensions are treated as 1 if not overridden - for (int64_t& dim : output_shape) { - if (dim == -1) { - dim = 1; + // free dimensions are treated as 1 if not overridden + for (int64_t& dim : output_shape) { + if (dim == -1) { + dim = 1; + } } - } - outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), - output_shape.size(), tensor_info.GetElementType())); + outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); + } } } From 88445568d11980e6e28d21ada96930da19ad2d41 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Thu, 29 Aug 2024 22:34:01 -0700 Subject: [PATCH 17/23] remove redundant code --- include/onnxruntime/core/framework/allocator.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index ea92cee26a25e..614d82c98bcd3 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -50,7 +50,6 @@ constexpr const char* HIP = "Hip"; constexpr const char* HIP_PINNED = "HipPinned"; constexpr const char* OpenVINO_CPU = "OpenVINO_CPU"; constexpr const char* OpenVINO_GPU = "OpenVINO_GPU"; -constexpr const char* OpenVINO_NPU = "OpenVINO_RT_NPU"; // application // 1. Allocate with ORT::CreateTensor("") @@ -58,7 +57,6 @@ constexpr const char* OpenVINO_NPU = "OpenVINO_RT_NPU"; constexpr const char* OpenVINO_RT = "OpenVINO_RT"; constexpr const char* OpenVINO_RT_NPU = "OpenVINO_RT_NPU"; -constexpr const char* WIN32_HANDLE = "WIN32_HANDLE"; constexpr const char* WEBGPU_BUFFER = "WebGPU_Buffer"; constexpr size_t kAllocAlignment = 256; From 8b32612604cc6ee27345871baa9a4467bdf4e022 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Fri, 30 Aug 2024 04:31:09 -0700 Subject: [PATCH 18/23] remove redundant statements --- include/onnxruntime/core/framework/allocator.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 614d82c98bcd3..abab118efd04f 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -50,11 +50,6 @@ constexpr const char* HIP = "Hip"; constexpr const char* HIP_PINNED = "HipPinned"; constexpr const char* OpenVINO_CPU = "OpenVINO_CPU"; constexpr const char* OpenVINO_GPU = "OpenVINO_GPU"; - -// application -// 1. Allocate with ORT::CreateTensor("") -// 2. "Manual" allocation - constexpr const char* OpenVINO_RT = "OpenVINO_RT"; constexpr const char* OpenVINO_RT_NPU = "OpenVINO_RT_NPU"; constexpr const char* WEBGPU_BUFFER = "WebGPU_Buffer"; From 4830c896ca7cf8ea6ca54a5359563de9ad36e986 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Mon, 2 Sep 2024 04:53:04 -0700 Subject: [PATCH 19/23] fix crash during inference --- .../core/providers/openvino/backends/basic_backend.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 15ded4965bcc1..2d3028af23893 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -295,8 +295,8 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ov_tensor_key = it->second; } else { // Does this make sense for both types of allocators? - auto input = ie_cnn_network_->get_parameters().at(input_idx); - ov_tensor_key.tensor_ptr = std::make_shared(input->get_element_type(), input->get_shape(), + auto input = graph_input_info.at(input_idx); + ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), (void*)tensor.GetTensorRawData()); if (allocator_name == OpenVINO_RT_NPU) { ov_tensor_key.copy_needed = false; @@ -361,8 +361,8 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { ov_tensor_data = it->second; } else { - auto output = ie_cnn_network_->get_results().at(output_idx); - ov_tensor_data.tensor_ptr = std::make_shared(output->get_element_type(), output->get_shape(), + auto output = graph_output_info.at(output_idx); + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), (void*)tensor.GetTensorRawData()); if(allocator_name == OpenVINO_RT_NPU) { ov_tensor_data.copy_needed = false; From 48c5569d0e5f9a1add05a8623dc4f3cc13f0fedb Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Mon, 2 Sep 2024 22:35:58 -0700 Subject: [PATCH 20/23] remove redundant code --- .../core/providers/openvino/backends/basic_backend.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 2d3028af23893..e6041f9190522 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -341,13 +341,6 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque break; } } - if (!output_name_found) { - ORT_THROW( - log_tag + - "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + - onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names"); - } - size_t batch_size = 1; Ort::UnownedValue tensor = GetOutputTensor(context, batch_size, From 1791b907651e3fca18d7257f97487e463a3303cb Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Wed, 4 Sep 2024 06:42:46 -0700 Subject: [PATCH 21/23] enable backward compatibility of remote tensor feature --- .../core/providers/openvino/backends/basic_backend.cc | 10 +++++----- .../providers/openvino/openvino_execution_provider.cc | 3 ++- .../providers/openvino/openvino_execution_provider.h | 4 ++-- onnxruntime/core/providers/openvino/ov_allocator.cc | 3 ++- onnxruntime/core/providers/openvino/ov_allocator.h | 3 ++- onnxruntime/test/perftest/ort_test_session.cc | 4 +--- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index e6041f9190522..46ea247ea03eb 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -287,8 +287,8 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + Ort::ConstValue tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + std::string allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); ov_tensor_data_t ov_tensor_key; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { @@ -315,7 +315,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque if (ov_tensor_key.copy_needed) { const char* ort_tensor_data = tensor.GetTensorData(); size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; + const char *ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); } } @@ -324,7 +324,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque // Set the output blob as remote blob auto graph_output_info = exe_network_.Get().outputs(); - auto output_idx = 0; + int output_idx = 0; for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { auto output_names = output_info_iter->get_names(); @@ -347,7 +347,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque infer_request, output_name, subgraph_context_.output_names); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + std::string allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); ov_tensor_data_t ov_tensor_data; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 534b05a352310..9717f2039ea9c 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -180,7 +180,7 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } - +#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info { [this](OrtDevice::DeviceId device_id) { @@ -192,5 +192,6 @@ std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() // fill in allocator return std::vector{CreateAllocator(npu_allocator_info)}; } +#endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 5f8f048ccd4ad..2e6d8a857de84 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,9 +189,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } - +#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 std::vector CreatePreferredAllocators() override; - +#endif private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 6a6939b74202e..2a702ed5a64e2 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License - +#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" @@ -52,3 +52,4 @@ void OVRTAllocator::Free(void* p) { } } // namespace onnxruntime +#endif diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 46a3afdf2274b..5e2298ccdb936 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License - +#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 #pragma once #include "core/common/inlined_containers.h" @@ -22,3 +22,4 @@ class OVRTAllocator : public IAllocator { }; } // namespace onnxruntime +#endif diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index e560da64af73e..35377ff7809b9 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -976,9 +976,6 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - if (!use_device_mem){ - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - } std::vector input_node_dim = tensor_info.GetShape(); // free dimensions are treated as 1 if not overridden @@ -993,6 +990,7 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } else { + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); auto allocator = Ort::AllocatorWithDefaultOptions(); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), input_node_dim.size(), tensor_info.GetElementType()); From f6f439bddd55ce762c6260c7f6579ed1fe914a50 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Wed, 4 Sep 2024 22:45:38 -0700 Subject: [PATCH 22/23] Revert "enable backward compatibility of remote tensor feature" This reverts commit 1791b907651e3fca18d7257f97487e463a3303cb. --- .../core/providers/openvino/backends/basic_backend.cc | 10 +++++----- .../providers/openvino/openvino_execution_provider.cc | 3 +-- .../providers/openvino/openvino_execution_provider.h | 4 ++-- onnxruntime/core/providers/openvino/ov_allocator.cc | 3 +-- onnxruntime/core/providers/openvino/ov_allocator.h | 3 +-- onnxruntime/test/perftest/ort_test_session.cc | 4 +++- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 46ea247ea03eb..e6041f9190522 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -287,8 +287,8 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - Ort::ConstValue tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - std::string allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); ov_tensor_data_t ov_tensor_key; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { @@ -315,7 +315,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque if (ov_tensor_key.copy_needed) { const char* ort_tensor_data = tensor.GetTensorData(); size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); - const char *ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; + auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); } } @@ -324,7 +324,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque // Set the output blob as remote blob auto graph_output_info = exe_network_.Get().outputs(); - int output_idx = 0; + auto output_idx = 0; for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { auto output_names = output_info_iter->get_names(); @@ -347,7 +347,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque infer_request, output_name, subgraph_context_.output_names); - std::string allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); ov_tensor_data_t ov_tensor_data; ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 9717f2039ea9c..534b05a352310 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -180,7 +180,7 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } -#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 + std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info { [this](OrtDevice::DeviceId device_id) { @@ -192,6 +192,5 @@ std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() // fill in allocator return std::vector{CreateAllocator(npu_allocator_info)}; } -#endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 2e6d8a857de84..5f8f048ccd4ad 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,9 +189,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } -#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 + std::vector CreatePreferredAllocators() override; -#endif + private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 2a702ed5a64e2..6a6939b74202e 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 + #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" @@ -52,4 +52,3 @@ void OVRTAllocator::Free(void* p) { } } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 5e2298ccdb936..46a3afdf2274b 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#if OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 4 + #pragma once #include "core/common/inlined_containers.h" @@ -22,4 +22,3 @@ class OVRTAllocator : public IAllocator { }; } // namespace onnxruntime -#endif diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 35377ff7809b9..e560da64af73e 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -976,6 +976,9 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + if (!use_device_mem){ + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + } std::vector input_node_dim = tensor_info.GetShape(); // free dimensions are treated as 1 if not overridden @@ -990,7 +993,6 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } else { - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); auto allocator = Ort::AllocatorWithDefaultOptions(); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), input_node_dim.size(), tensor_info.GetElementType()); From 39c0cba9b9033707d663ee2fab4d696cc879e8f4 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Wed, 4 Sep 2024 23:18:35 -0700 Subject: [PATCH 23/23] enable backward compatibility of remote tensor feature in OVEP --- cmake/onnxruntime_providers_openvino.cmake | 4 ++++ .../core/providers/openvino/openvino_execution_provider.cc | 4 ++++ .../core/providers/openvino/openvino_execution_provider.h | 4 ++-- onnxruntime/core/providers/openvino/ov_allocator.cc | 3 ++- onnxruntime/core/providers/openvino/ov_allocator.h | 3 ++- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index e559583fae8f5..69805d60d4593 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -21,6 +21,10 @@ message(FATAL_ERROR "OpenVINO 2024.0 and newer are supported. Please, use latest OpenVINO release") endif() + if(OpenVINO_VERSION VERSION_GREATER_EQUAL 2024.4) + add_definitions(-DUSE_DEVICE_MEMORY=1) + endif() + if (WIN32) unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) endif() diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 534b05a352310..4f9764212f37d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,7 +10,9 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" +#ifdef USE_DEVICE_MEMORY #include "core/providers/openvino/ov_allocator.h" +#endif #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) @@ -181,6 +183,7 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } +#ifdef USE_DEVICE_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info { [this](OrtDevice::DeviceId device_id) { @@ -192,5 +195,6 @@ std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() // fill in allocator return std::vector{CreateAllocator(npu_allocator_info)}; } +#endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 5f8f048ccd4ad..42a8368a57e29 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,9 +189,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } - +#ifdef USE_DEVICE_MEMORY std::vector CreatePreferredAllocators() override; - +#endif private: std::unique_ptr global_context_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 6a6939b74202e..c7f22039a8b0e 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License - +#ifdef USE_DEVICE_MEMORY #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" @@ -52,3 +52,4 @@ void OVRTAllocator::Free(void* p) { } } // namespace onnxruntime +#endif diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 46a3afdf2274b..4a02f0013beac 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License - +#ifdef USE_DEVICE_MEMORY #pragma once #include "core/common/inlined_containers.h" @@ -22,3 +22,4 @@ class OVRTAllocator : public IAllocator { }; } // namespace onnxruntime +#endif