From 5011befba9465867c57a869061ed252a05893cad Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Mon, 9 Sep 2024 23:08:20 -0700 Subject: [PATCH 1/3] fix debug build issue and lint issues --- .../openvino/backends/basic_backend.cc | 6 ++---- .../openvino/backends/basic_backend.h | 2 +- .../openvino/openvino_execution_provider.cc | 8 +++---- .../core/providers/openvino/ov_allocator.cc | 2 +- .../core/providers/openvino/ov_allocator.h | 7 +++---- onnxruntime/test/perftest/ort_test_session.cc | 21 +++++++++---------- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d274bbc64558d..0431fe1905405 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -302,7 +302,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque // Does this make sense for both types of allocators? auto input = graph_input_info.at(input_idx); ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), - (void*)tensor.GetTensorRawData()); + (void*)tensor.GetTensorRawData()); if (allocator_name == OpenVINO_RT_NPU) { ov_tensor_key.copy_needed = false; } else { @@ -335,14 +335,12 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; - bool output_name_found = false; // using the output name retrieved from ONNX original to match with the output names returned by OV tensors for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { onnx_output_name = it->first; if (output_names.find(onnx_output_name) != output_names.end()) { // Assigning the output_name output_name = it->first; - output_name_found = true; break; } } @@ -362,7 +360,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque auto output = graph_output_info.at(output_idx); ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), (void*)tensor.GetTensorRawData()); - if(allocator_name == OpenVINO_RT_NPU) { + if (allocator_name == OpenVINO_RT_NPU) { ov_tensor_data.copy_needed = false; } else { ov_tensor_data.copy_needed = true; diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index 4f430abe27b6b..cd69e88f994b9 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -67,7 +67,7 @@ class BasicBackend : public IBackend { OVRemoteContextPtr remote_context_; #endif - using ort_tensor_key_t = std::pair; + using ort_tensor_key_t = std::pair; std::map ort_ov_tensor_map; }; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 4f9764212f37d..30d49af07eaa3 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -185,11 +185,11 @@ common::Status OpenVINOExecutionProvider::Compile( #ifdef USE_DEVICE_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { - AllocatorCreationInfo npu_allocator_info { - [this](OrtDevice::DeviceId device_id) { + AllocatorCreationInfo npu_allocator_info{ + [this](OrtDevice::DeviceId device_id) { return std::make_unique(global_context_->ie_core.Get(), OrtDevice::NPU, device_id, OpenVINO_RT_NPU); - }, - 0, + }, + 0, }; // fill in allocator diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index c7f22039a8b0e..48ed626981dfb 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -28,7 +28,7 @@ void* OVRTAllocator::Alloc(size_t size) { try { size_t alloc_size = align_up(size + sizeof(ov::Tensor*) + default_alignment, default_alignment); ov::Tensor* tensor = new ov::Tensor(remote_ctx_.create_host_tensor(ov::element::Type_t::u8, - { alloc_size })); + {alloc_size})); uintptr_t data_ptr = reinterpret_cast(tensor->data()); ov::Tensor** ptr = reinterpret_cast(align_up(data_ptr + sizeof(ov::Tensor*), default_alignment)); diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 4a02f0013beac..24d61734a3c0b 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -7,18 +7,17 @@ #include "core/framework/allocator.h" #include "openvino/runtime/remote_context.hpp" - namespace onnxruntime { class OVRTAllocator : public IAllocator { public: - OVRTAllocator(ov::Core &core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); + OVRTAllocator(ov::Core& core, OrtDevice::DeviceType device_type, OrtDevice::DeviceId device_id, const char* name); void* Alloc(size_t size) override; void Free(void* p) override; private: - ov::Core &core_; - ov::RemoteContext remote_ctx_; + ov::Core& core_; + ov::RemoteContext remote_ctx_; }; } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 48312e2c42e5d..ae7680571ced1 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -40,10 +40,10 @@ std::chrono::duration OnnxRuntimeTestSession::Run() { if (!use_device_mem) { auto output_values = session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), output_names_raw_ptr.size()); + output_names_raw_ptr.data(), output_names_raw_ptr.size()); } else { - session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), - output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); + session_.Run(Ort::RunOptions{nullptr}, input_names_.data(), input.data(), input_names_.size(), + output_names_raw_ptr.data(), outputs_.data(), output_names_raw_ptr.size()); } auto end = std::chrono::high_resolution_clock::now(); @@ -827,8 +827,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); if (value == "true" || value == "True") { use_device_mem = true; } - } - else { + } else { ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } @@ -888,8 +887,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } } - outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), - output_shape.size(), tensor_info.GetElementType())); + outputs_.push_back(Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)output_shape.data(), + output_shape.size(), tensor_info.GetElementType())); } } } @@ -980,7 +979,7 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - if (!use_device_mem){ + if (!use_device_mem) { Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); } std::vector input_node_dim = tensor_info.GetShape(); @@ -991,15 +990,15 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { dim = 1; } } - if (use_device_mem){ + if (use_device_mem) { Ort::Value input_tensor = Ort::Value::CreateTensor(*custom_allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); + input_node_dim.size(), tensor_info.GetElementType()); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } else { auto allocator = Ort::AllocatorWithDefaultOptions(); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); + input_node_dim.size(), tensor_info.GetElementType()); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } From 068e1311055bbcdeab4b2d94629478fc45c0ddb3 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Tue, 10 Sep 2024 07:47:15 -0700 Subject: [PATCH 2/3] change naming for OVEP NPU specific macro --- cmake/onnxruntime_providers_openvino.cmake | 2 +- .../core/providers/openvino/openvino_execution_provider.cc | 4 ++-- .../core/providers/openvino/openvino_execution_provider.h | 2 +- onnxruntime/core/providers/openvino/ov_allocator.cc | 2 +- onnxruntime/core/providers/openvino/ov_allocator.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 69805d60d4593..2eb3611bae902 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -22,7 +22,7 @@ endif() if(OpenVINO_VERSION VERSION_GREATER_EQUAL 2024.4) - add_definitions(-DUSE_DEVICE_MEMORY=1) + add_definitions(-DUSE_OVEP_NPU_MEMORY=1) endif() if (WIN32) diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 30d49af07eaa3..08144651319cf 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -10,7 +10,7 @@ #include "core/providers/openvino/onnx_ctx_model_helper.h" #include "core/providers/openvino/ov_versions/capability.h" #include "openvino/core/version.hpp" -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #endif @@ -183,7 +183,7 @@ common::Status OpenVINOExecutionProvider::Compile( return Status::OK(); } -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() { AllocatorCreationInfo npu_allocator_info{ [this](OrtDevice::DeviceId device_id) { diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 42a8368a57e29..8b1c62c607f6e 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -189,7 +189,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { const void* GetExecutionHandle() const noexcept override { return nullptr; } -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY std::vector CreatePreferredAllocators() override; #endif private: diff --git a/onnxruntime/core/providers/openvino/ov_allocator.cc b/onnxruntime/core/providers/openvino/ov_allocator.cc index 48ed626981dfb..6700244b754d8 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.cc +++ b/onnxruntime/core/providers/openvino/ov_allocator.cc @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #include "core/providers/openvino/ov_allocator.h" #include "core/providers/openvino/ov_interface.h" #include "openvino/runtime/intel_npu/level_zero/level_zero.hpp" diff --git a/onnxruntime/core/providers/openvino/ov_allocator.h b/onnxruntime/core/providers/openvino/ov_allocator.h index 24d61734a3c0b..083cfc4d5aed3 100644 --- a/onnxruntime/core/providers/openvino/ov_allocator.h +++ b/onnxruntime/core/providers/openvino/ov_allocator.h @@ -1,6 +1,6 @@ // Copyright (C) Intel Corporation // Licensed under the MIT License -#ifdef USE_DEVICE_MEMORY +#ifdef USE_OVEP_NPU_MEMORY #pragma once #include "core/common/inlined_containers.h" From 66e4d898e4ba981cea028eb592d52f0cdd1700a4 Mon Sep 17 00:00:00 2001 From: saurabhkale17 Date: Wed, 11 Sep 2024 07:29:33 -0700 Subject: [PATCH 3/3] fix unit tests and lint issues --- .../openvino/backends/basic_backend.cc | 201 ++++++++++-------- 1 file changed, 117 insertions(+), 84 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 0431fe1905405..1f9c61780f27a 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -292,88 +292,102 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque ORT_THROW(msg); } } else { - auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_key; - ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_key = it->second; - } else { - // Does this make sense for both types of allocators? - auto input = graph_input_info.at(input_idx); - ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), - (void*)tensor.GetTensorRawData()); - if (allocator_name == OpenVINO_RT_NPU) { - ov_tensor_key.copy_needed = false; - } else { - ov_tensor_key.copy_needed = true; - } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); - + if ((global_context_.device_type.find("CPU") != std::string::npos || + global_context_.device_type.find("GPU") != std::string::npos)) { + OVTensorPtr graph_input_blob; try { - infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + graph_input_blob = infer_request->GetTensor(input_name); } catch (const char* msg) { ORT_THROW(msg); } - } - - if (ov_tensor_key.copy_needed) { - const char* ort_tensor_data = tensor.GetTensorData(); - size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; - std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); + FillInputBlob(std::move(graph_input_blob), batch_slice_idx, std::move(input_name), context, subgraph_context_); + } else { + auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_key; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_key = it->second; + } else { + // Does this make sense for both types of allocators? + auto input = graph_input_info.at(input_idx); + if (allocator_name == OpenVINO_RT_NPU) { + ov_tensor_key.copy_needed = false; + ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape(), + (void*)tensor.GetTensorRawData()); + } else { + ov_tensor_key.copy_needed = true; + ov_tensor_key.tensor_ptr = std::make_shared(input.get_element_type(), input.get_shape()); + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_key); + + if (ov_tensor_key.copy_needed) { + const char* ort_tensor_data = tensor.GetTensorData(); + size_t tensor_data_size = ov_tensor_key.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data + tensor_data_size * batch_slice_idx; + std::memcpy(ov_tensor_key.tensor_ptr->data(), ort_batch_memory_offset, tensor_data_size); + } + + try { + infer_request->SetTensor(input_name, ov_tensor_key.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } + } } } input_idx++; } - - // Set the output blob as remote blob - auto graph_output_info = exe_network_.Get().outputs(); - auto output_idx = 0; - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - auto output_names = output_info_iter->get_names(); - std::string onnx_output_name; - std::string output_name; - // using the output name retrieved from ONNX original to match with the output names returned by OV tensors - for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { - onnx_output_name = it->first; - if (output_names.find(onnx_output_name) != output_names.end()) { - // Assigning the output_name - output_name = it->first; - break; + if (global_context_.device_type.find("NPU") != std::string::npos) { + // Set the output blob as remote blob + auto graph_output_info = exe_network_.Get().outputs(); + auto output_idx = 0; + for (auto output_info_iter = graph_output_info.begin(); + output_info_iter != graph_output_info.end(); ++output_info_iter) { + auto output_names = output_info_iter->get_names(); + std::string onnx_output_name; + std::string output_name; + // using the output name retrieved from ONNX original to match with the output names returned by OV tensors + for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { + onnx_output_name = it->first; + if (output_names.find(onnx_output_name) != output_names.end()) { + // Assigning the output_name + output_name = it->first; + break; + } } - } - size_t batch_size = 1; - Ort::UnownedValue tensor = GetOutputTensor(context, - batch_size, - infer_request, - output_name, - subgraph_context_.output_names); - auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); + size_t batch_size = 1; + Ort::UnownedValue tensor = GetOutputTensor(context, + batch_size, + infer_request, + output_name, + subgraph_context_.output_names); + auto allocator_name = tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_data; - ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; - } else { - auto output = graph_output_info.at(output_idx); - ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), - (void*)tensor.GetTensorRawData()); - if (allocator_name == OpenVINO_RT_NPU) { - ov_tensor_data.copy_needed = false; + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; } else { - ov_tensor_data.copy_needed = true; - } - ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); + auto output = graph_output_info.at(output_idx); + if (allocator_name == OpenVINO_RT_NPU) { + ov_tensor_data.copy_needed = false; + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape(), + (void*)tensor.GetTensorRawData()); + } else { + ov_tensor_data.copy_needed = true; + ov_tensor_data.tensor_ptr = std::make_shared(output.get_element_type(), output.get_shape()); + } + ort_ov_tensor_map.emplace(ort_tensor_key, ov_tensor_data); - try { - infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); - } catch (const char* msg) { - ORT_THROW(msg); + try { + infer_request->SetTensor(output_name, ov_tensor_data.tensor_ptr); + } catch (const char* msg) { + ORT_THROW(msg); + } } + output_idx++; } - output_idx++; } // Start Async inference @@ -501,6 +515,7 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe auto graph_output_info = exe_network_.Get().outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { + OVTensorPtr graph_output_blob; auto output_names = output_info_iter->get_names(); std::string onnx_output_name; std::string output_name; @@ -524,24 +539,42 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe " doesn't exist in the " "list of OpenVINO output tensor names"); } - - size_t batch_size = 1; - Ort::UnownedValue output_tensor = - GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); - auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); - ov_tensor_data_t ov_tensor_data; - ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; - if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { - ov_tensor_data = it->second; + if ((global_context_.device_type.find("CPU") != std::string::npos || + global_context_.device_type.find("GPU") != std::string::npos)) { + try { + graph_output_blob = infer_request->GetTensor(output_name); + } catch (const char* msg) { + ORT_THROW(msg); + } + size_t batch_size = 1; + Ort::UnownedValue output_tensor = + GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); + auto mem_info = output_tensor.GetTensorMemoryInfo(); + if (mem_info.GetAllocatorName() == OpenVINO_GPU) { + return; + } else { + size_t batch_slice = 0; + FillOutputBlob(std::move(graph_output_blob), output_tensor, batch_slice); + } } else { - ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); - } + size_t batch_size = 1; + Ort::UnownedValue output_tensor = + GetOutputTensor(context, batch_size, infer_request, std::move(output_name), subgraph_context_.output_names); + auto allocator_name = output_tensor.GetTensorMemoryInfo().GetAllocatorName(); + ov_tensor_data_t ov_tensor_data; + ort_tensor_key_t ort_tensor_key{output_tensor.GetTensorRawData(), allocator_name}; + if (const auto& it = ort_ov_tensor_map.find(ort_tensor_key); it != ort_ov_tensor_map.end()) { + ov_tensor_data = it->second; + } else { + ORT_THROW(log_tag + "Expected all outputs to have associated OV::Tensor's"); + } - if (ov_tensor_data.copy_needed) { - auto ort_tensor_data = output_tensor.GetTensorMutableData(); - size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); - auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; - std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); + if (ov_tensor_data.copy_needed) { + auto ort_tensor_data = output_tensor.GetTensorMutableData(); + size_t tensor_data_size = ov_tensor_data.tensor_ptr->get_byte_size(); + auto ort_batch_memory_offset = ort_tensor_data /*+ tensor_data_size * batch_size*/; + std::memcpy(ort_batch_memory_offset, ov_tensor_data.tensor_ptr->data(), tensor_data_size); + } } }