diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index 4fca4037301fb..96ad98491f902 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -502,6 +502,15 @@ void BackendManager::Compute(OrtKernelContext* context) { dynamic_backend->Infer(context); } else { + if (dynamic_workload_type == "EFFICIENT") { + auto ov_execution_network = concrete_backend_->GetOVCompiledModel(); + ov_execution_network.set_property({{"WORKLOAD_TYPE", "EFFICIENT"}}); + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << " OV WORKLOAD_TYPE property is set to EFFICIENT mode for OV inference"; + } else if (dynamic_workload_type == "DEFAULT") { + auto ov_execution_network = concrete_backend_->GetOVCompiledModel(); + ov_execution_network.set_property({{"WORKLOAD_TYPE", "DEFAULT"}}); + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << " OV WORKLOAD_TYPE property is set to DEFAULT mode for OV inference"; + } concrete_backend_->Infer(context); } #ifdef OPENVINO_FIL_ENABLED diff --git a/onnxruntime/core/providers/openvino/backend_manager.h b/onnxruntime/core/providers/openvino/backend_manager.h index b9ff7a72372b3..f1f5c1ef04025 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.h +++ b/onnxruntime/core/providers/openvino/backend_manager.h @@ -30,6 +30,7 @@ class BackendManager { GlobalContext& GetGlobalContext(); Status ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& subgraph, const logging::Logger& logger); + std::string dynamic_workload_type = "DEFAULT"; private: std::unique_ptr GetModelProtoFromFusedNode( diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 19a634818a442..77f19dda37c19 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -150,7 +150,7 @@ common::Status OpenVINOExecutionProvider::Compile( graph_body_viewer, *GetLogger(), ep_ctx_handle_); - + backend_manager_ = backend_manager; compute_info.create_state_func = [backend_manager](ComputeContext* context, FunctionState* state) { OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState(); @@ -198,4 +198,18 @@ std::vector OpenVINOExecutionProvider::CreatePreferredAllocators() } #endif +common::Status OpenVINOExecutionProvider::SetEpDynamicOptions(gsl::span keys, + gsl::span values) { + if ((std::find(keys.begin(), keys.end(), "ep.dynamic.workload_type") != keys.end()) && + (std::find(values.begin(), values.end(), "Efficient") != values.end())) { + backend_manager_->dynamic_workload_type = "EFFICIENT"; + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << backend_manager_->dynamic_workload_type << " mode is set for OV inference"; + } + if ((std::find(keys.begin(), keys.end(), "ep.dynamic.workload_type") != keys.end()) && + (std::find(values.begin(), values.end(), "Default") != values.end())) { + backend_manager_->dynamic_workload_type = "DEFAULT"; + LOGS_DEFAULT(VERBOSE) << "[OpenVINO-EP]" << backend_manager_->dynamic_workload_type << " mode is set for OV inference"; + } + return Status::OK(); +} } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 7d9da65ea7e07..bea9badea475a 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -188,6 +188,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider { Status Compile(const std::vector& fused_nodes, std::vector& node_compute_funcs) override; + Status SetEpDynamicOptions(gsl::span /*keys*/, + gsl::span /*values*/) override; + const void* GetExecutionHandle() const noexcept override { return nullptr; } @@ -196,6 +199,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider { #endif private: std::unique_ptr global_context_; + std::shared_ptr backend_manager_; openvino_ep::EPCtxHandler ep_ctx_handle_{}; };