From 11e7ee50bacb51d37307b91917d07294a5e24db6 Mon Sep 17 00:00:00 2001 From: ynimmaga Date: Tue, 5 Nov 2024 20:46:36 -0800 Subject: [PATCH 1/2] First commit for openvino backend --- backends/openvino/CMakeLists.txt | 52 +++++++ backends/openvino/__init__.py | 4 + backends/openvino/partitioner.py | 112 +++++++++++++++ backends/openvino/preprocess.py | 45 ++++++ backends/openvino/requirements.txt | 8 ++ backends/openvino/runtime/OpenvinoBackend.cpp | 134 ++++++++++++++++++ 6 files changed, 355 insertions(+) create mode 100644 backends/openvino/CMakeLists.txt create mode 100644 backends/openvino/__init__.py create mode 100644 backends/openvino/partitioner.py create mode 100644 backends/openvino/preprocess.py create mode 100644 backends/openvino/requirements.txt create mode 100644 backends/openvino/runtime/OpenvinoBackend.cpp diff --git a/backends/openvino/CMakeLists.txt b/backends/openvino/CMakeLists.txt new file mode 100644 index 00000000000..9bb67fc97eb --- /dev/null +++ b/backends/openvino/CMakeLists.txt @@ -0,0 +1,52 @@ +cmake_minimum_required(VERSION 3.19) +project(openvino_backend) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Source root directory for executorch. +if(NOT EXECUTORCH_ROOT) + set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) +endif() + +include(${EXECUTORCH_ROOT}/build/Utils.cmake) + +set(_common_include_directories ${EXECUTORCH_ROOT}/..) + +# Set openvino directory from environment +set(OPENVINO_DIR "$ENV{INTEL_OPENVINO_DIR}") +set(OPENVINO_INCLUDE_DIRS ${OPENVINO_DIR}/deployment_tools/inference_engine/include ${OPENVINO_DIR}/runtime/include) +message("${OPENVINO_DIR}/runtime/include/openvino") + +# Define the source files for the OpenVINO backend +set(_openvino_backend_sources backends/openvino/runtime/OpenvinoBackend.cpp) + +list(TRANSFORM _openvino_backend_sources PREPEND "${EXECUTORCH_ROOT}/") + +# Add the OpenVINO backend library +add_library(openvino_backend STATIC ${_openvino_backend_sources}) + +# Include directories for ExecuteTorch and OpenVINO +target_include_directories( + openvino_backend PUBLIC ${_common_include_directories} +) + +target_include_directories( + openvino_backend PUBLIC ${OPENVINO_INCLUDE_DIRS} +) + +set(OPENVINO_LIB_PATH ${OPENVINO_DIR}/runtime/lib/intel64) +set(OPENVINO_LIBS + ${OPENVINO_LIB_PATH}/libopenvino.so + ${OPENVINO_LIB_PATH}/libopenvino_ir_frontend.so.2430 + ${OPENVINO_LIB_PATH}/libopenvino_c.so + ${OPENVINO_LIB_PATH}/libopenvino_intel_cpu_plugin.so + ${OPENVINO_LIB_PATH}/libopenvino_intel_gpu_plugin.so + ${OPENVINO_LIB_PATH}/libopenvino_auto_plugin.so +) + +# Link the OpenVINO library to the backend +target_link_libraries(openvino_backend PRIVATE OPENVINO_LIBS) + diff --git a/backends/openvino/__init__.py b/backends/openvino/__init__.py new file mode 100644 index 00000000000..dac275d3f12 --- /dev/null +++ b/backends/openvino/__init__.py @@ -0,0 +1,4 @@ +from .partitioner import OpenvinoPartitioner +from .preprocess import OpenvinoBackend + +__all__ = [OpenvinoBackend, OpenvinoPartitioner] diff --git a/backends/openvino/partitioner.py b/backends/openvino/partitioner.py new file mode 100644 index 00000000000..2fa20bd8831 --- /dev/null +++ b/backends/openvino/partitioner.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024 MediaTek Inc. +# +# Licensed under the BSD License (the "License"); you may not use this file +# except in compliance with the License. See the license file in the root +# directory of this source tree for more details. + +from typing import Callable, final, List, Optional, Tuple + +import torch +from executorch.backends.openvino.preprocess import OpenvinoBackend +from executorch.exir.backend.backend_details import CompileSpec +from executorch.exir.backend.partitioner import ( + DelegationSpec, + Partitioner, + PartitionResult, +) +from executorch.exir.backend.utils import tag_constant_data + +from torch.export.exported_program import ExportedProgram +from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner +from torch.fx.passes.operator_support import OperatorSupportBase +import torch.fx as fx +from openvino.frontend.pytorch.torchdynamo.op_support import OperatorSupport + +class OpenvinoOperatorsSupport(OperatorSupportBase): + + def __init__( + self, + op_types_to_skip: Optional[set] = None, + op_names_to_skip: Optional[set] = None, + ) -> None: + if op_types_to_skip is None: + op_types_to_skip = set() + if op_names_to_skip is None: + op_names_to_skip = set() + + self._op_types_to_skip = op_types_to_skip + self._op_names_to_skip = op_names_to_skip + + def is_node_supported(self, _, node: torch.fx.Node) -> bool: + if node.op != "call_function": + return False + + options = [] + op_type = node.target.__name__ + supported_ops = OperatorSupport(options)._support_dict + if (op_type == "getitem"): + return True + + if ("torch.ops." + str(op_type) in supported_ops): + return True + else: + print("Op not supported: ", "torch.ops." + str(op_type)) + + if op_type in self._op_types_to_skip or node.name in self._op_names_to_skip: + print( + f"[OpenVINO Backend] The {op_type} operator with name '{node.name}' is skipped." + ) + return False + + return False + + +@final +class OpenvinoPartitioner(Partitioner): + + def __init__( + self, + compile_spec: List[CompileSpec], + op_types_to_skip: Optional[set] = None, + op_names_to_skip: Optional[set] = None, + ) -> None: + self.delegation_spec = DelegationSpec(OpenvinoBackend.__name__, compile_spec) + self._op_types_to_skip = op_types_to_skip + self._op_names_to_skip = op_names_to_skip + + def ops_to_not_decompose( + self, + ep: ExportedProgram, + ) -> Tuple[List[torch._ops.OpOverload], Optional[Callable[[torch.fx.Node], bool]]]: + ops_not_decompose = [ + torch.ops.aten.pixel_shuffle.default, + torch.ops.aten.upsample_bilinear2d.default, + torch.ops.aten.upsample_bilinear2d.vec, + torch.ops.aten.upsample_nearest2d.default, + torch.ops.aten.upsample_nearest2d.vec, + ] + return (ops_not_decompose, None) + + def partition(self, exported_program: ExportedProgram) -> PartitionResult: + options = {} + gm = fx.symbolic_trace(exported_program.graph_module) + + partitioner = CapabilityBasedPartitioner( + exported_program.graph_module, + OpenvinoOperatorsSupport(self._op_types_to_skip, self._op_names_to_skip), + allows_single_node_partition=True + ) + partition_list = partitioner.propose_partitions() + + partition_tags = {} + for partition in partition_list: + for node in partition.nodes: + tag = f"tag{partition.id}" + node.meta["delegation_tag"] = tag + partition_tags[tag] = self.delegation_spec + + tag_constant_data(exported_program) + + return PartitionResult( + tagged_exported_program=exported_program, partition_tags=partition_tags + ) diff --git a/backends/openvino/preprocess.py b/backends/openvino/preprocess.py new file mode 100644 index 00000000000..bfb38474797 --- /dev/null +++ b/backends/openvino/preprocess.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024 MediaTek Inc. +# +# Licensed under the BSD License (the "License"); you may not use this file +# except in compliance with the License. See the license file in the root +# directory of this source tree for more details. + +import contextlib +import struct + +from typing import final, List, cast + +import torch +from executorch.exir.backend.backend_details import ( + BackendDetails, + ExportedProgram, + PreprocessResult, +) +from executorch.exir.backend.compile_spec_schema import CompileSpec +from openvino.frontend.pytorch.torchdynamo.compile import openvino_compile + +SKIP_COMPILE_SPEC_KEYS = {"ImportForever"} + + +@final +class OpenvinoBackend(BackendDetails): + + @classmethod + def preprocess( + cls, edge_program: ExportedProgram, module_compile_spec: List[CompileSpec] + ) -> PreprocessResult: + name_to_node_mappings = {node.name: node for node in edge_program.graph.nodes} + input_names = edge_program.graph_signature.user_inputs + output_names = edge_program.graph_signature.user_outputs + args = [] + for node in edge_program.graph.nodes: + if (node.target in input_names): + args.append( node.meta["val"]) + + input_shapes = [] + output_shapes = [] + + compiled = openvino_compile(edge_program.module(), *args) + model_bytes = compiled.export_model() + + return PreprocessResult(processed_bytes=model_bytes) diff --git a/backends/openvino/requirements.txt b/backends/openvino/requirements.txt new file mode 100644 index 00000000000..7c3de886e27 --- /dev/null +++ b/backends/openvino/requirements.txt @@ -0,0 +1,8 @@ +datasets +huggingface-hub +safetensors +sentencepiece +tokenizers +transformers +piq +pillow diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp new file mode 100644 index 00000000000..491335b43b4 --- /dev/null +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -0,0 +1,134 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include + +using namespace std; +using executorch::aten::ScalarType; +using executorch::runtime::ArrayRef; +using executorch::runtime::Backend; +using executorch::runtime::BackendExecutionContext; +using executorch::runtime::BackendInitContext; +using executorch::runtime::CompileSpec; +using executorch::runtime::DelegateHandle; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::FreeableBuffer; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::Result; + +namespace executorch { +namespace backends { +namespace openvino { + +typedef struct { + std::shared_ptr compiled_model; + std::shared_ptr infer_request; +} ExecutionHandle; + +class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { + public: + OpenvinoBackend() {} + + ~OpenvinoBackend() = default; + + virtual bool is_available() const override { + // Check if OpenVINO runtime is available + return true; + } + + Result init( + BackendInitContext& context, + FreeableBuffer* processed, + ArrayRef compile_specs) const override { + ET_LOG(Info, "OpenvinoBackend::init %p", processed->data()); + + ov::Core core; + const char* data_ptr = static_cast(processed->data()); + size_t data_size = processed->size(); // Use appropriate size function here + + // Copy data to a string or vector + std::string data_string(data_ptr, data_size); + + // Wrap the data in a stream + std::istringstream compiled_stream(data_string); + + auto compiled_model = core.import_model(compiled_stream, "CPU"); //target_device); + + // Allocate an infer request + std::shared_ptr infer_request = std::make_shared(compiled_model.create_infer_request()); + + // Allocate execution handle + MemoryAllocator* allocator = context.get_runtime_allocator(); + ExecutionHandle* handle = ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR(allocator, ExecutionHandle); + handle->compiled_model = std::make_shared(compiled_model); //compiled_model; + handle->infer_request = infer_request; + + return handle; + } + + Error execute( + BackendExecutionContext& context, + DelegateHandle* input_handle, + EValue** args) const override { + ExecutionHandle* execution_handle = (ExecutionHandle*)input_handle; + + auto infer_request = execution_handle->infer_request; + + // Assume first argument is the input tensor + auto input_tensor = args[0]->toTensor(); + ov::Shape input_shape(input_tensor.sizes().begin(), input_tensor.sizes().end()); + + // Convert input tensor to OpenVINO tensor + ov::element::Type ov_type = convert_to_openvino_type(input_tensor.scalar_type()); + ov::Tensor ov_input_tensor(ov_type, input_shape, input_tensor.mutable_data_ptr()); + + infer_request->set_tensor("input", ov_input_tensor); + + // Execute the inference + infer_request->infer(); + + // Retrieve and copy output + auto output_tensor = args[1]->toTensor(); // Assume second argument is the output + ov::Tensor ov_output_tensor = infer_request->get_tensor("output"); + + std::memcpy(output_tensor.mutable_data_ptr(), ov_output_tensor.data(), ov_output_tensor.get_byte_size()); + + return Error::Ok; + } + + void destroy(DelegateHandle* handle) const override { + return; + } + + private: + ov::element::Type convert_to_openvino_type(ScalarType scalar_type) const { + // Convert ExecuteTorch scalar types to OpenVINO element types + switch (scalar_type) { + case ScalarType::Float: + return ov::element::f32; + case ScalarType::Int: + return ov::element::i32; + case ScalarType::Char: + return ov::element::i8; + default: + throw std::runtime_error("Unsupported scalar type"); + } + } +}; + +namespace { +auto backend = OpenvinoBackend(); +Backend backend_id{"OpenvinoBackend", &backend}; +static auto registered = register_backend(backend_id); +} // namespace + +} // namespace openvino +} // namespace backends +} // namespace executorch From be927aa4160a8893341d0ea2c31cc6d5b2e81213 Mon Sep 17 00:00:00 2001 From: ynimmaga Date: Mon, 11 Nov 2024 21:38:06 -0800 Subject: [PATCH 2/2] Added example for openvino backend --- backends/openvino/CMakeLists.txt | 30 ++-- backends/openvino/runtime/OpenvinoBackend.cpp | 29 ++-- examples/openvino/CMakeLists.txt | 83 ++++++++++ .../openvino_executor_runner.cpp | 145 ++++++++++++++++++ .../executor_runner/ov_executor_runner.cpp | 120 +++++++++++++++ examples/openvino/openvino_build.sh | 53 +++++++ 6 files changed, 436 insertions(+), 24 deletions(-) create mode 100644 examples/openvino/CMakeLists.txt create mode 100644 examples/openvino/executor_runner/openvino_executor_runner.cpp create mode 100644 examples/openvino/executor_runner/ov_executor_runner.cpp create mode 100755 examples/openvino/openvino_build.sh diff --git a/backends/openvino/CMakeLists.txt b/backends/openvino/CMakeLists.txt index 9bb67fc97eb..e6be4f14d79 100644 --- a/backends/openvino/CMakeLists.txt +++ b/backends/openvino/CMakeLists.txt @@ -1,11 +1,12 @@ -cmake_minimum_required(VERSION 3.19) -project(openvino_backend) - set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(_common_include_directories ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +include_directories(BEFORE ${_common_include_directories}) + # Source root directory for executorch. if(NOT EXECUTORCH_ROOT) set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) @@ -18,15 +19,10 @@ set(_common_include_directories ${EXECUTORCH_ROOT}/..) # Set openvino directory from environment set(OPENVINO_DIR "$ENV{INTEL_OPENVINO_DIR}") set(OPENVINO_INCLUDE_DIRS ${OPENVINO_DIR}/deployment_tools/inference_engine/include ${OPENVINO_DIR}/runtime/include) -message("${OPENVINO_DIR}/runtime/include/openvino") - -# Define the source files for the OpenVINO backend -set(_openvino_backend_sources backends/openvino/runtime/OpenvinoBackend.cpp) - -list(TRANSFORM _openvino_backend_sources PREPEND "${EXECUTORCH_ROOT}/") # Add the OpenVINO backend library -add_library(openvino_backend STATIC ${_openvino_backend_sources}) +add_library(openvino_backend SHARED) +target_compile_options(openvino_backend PRIVATE "-frtti" "-fexceptions") # Include directories for ExecuteTorch and OpenVINO target_include_directories( @@ -40,7 +36,7 @@ target_include_directories( set(OPENVINO_LIB_PATH ${OPENVINO_DIR}/runtime/lib/intel64) set(OPENVINO_LIBS ${OPENVINO_LIB_PATH}/libopenvino.so - ${OPENVINO_LIB_PATH}/libopenvino_ir_frontend.so.2430 + ${OPENVINO_LIB_PATH}/libopenvino_ir_frontend.so.2450 ${OPENVINO_LIB_PATH}/libopenvino_c.so ${OPENVINO_LIB_PATH}/libopenvino_intel_cpu_plugin.so ${OPENVINO_LIB_PATH}/libopenvino_intel_gpu_plugin.so @@ -48,5 +44,15 @@ set(OPENVINO_LIBS ) # Link the OpenVINO library to the backend -target_link_libraries(openvino_backend PRIVATE OPENVINO_LIBS) +target_link_libraries(openvino_backend PRIVATE ${OPENVINO_LIBS} executorch_core) + +target_sources( + openvino_backend + PRIVATE ${CMAKE_CURRENT_LIST_DIR}/runtime/OpenvinoBackend.cpp +) + +target_link_options_shared_lib(openvino_backend) +install(TARGETS openvino_backend DESTINATION lib) + + diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index 491335b43b4..baf2915e59d 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -34,7 +35,7 @@ typedef struct { class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { public: - OpenvinoBackend() {} + OpenvinoBackend() {std::cout << "In OV Backend constructor" << std::endl;} ~OpenvinoBackend() = default; @@ -50,8 +51,9 @@ class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { ET_LOG(Info, "OpenvinoBackend::init %p", processed->data()); ov::Core core; + const char* data_ptr = static_cast(processed->data()); - size_t data_size = processed->size(); // Use appropriate size function here + size_t data_size = processed->size(); // Copy data to a string or vector std::string data_string(data_ptr, data_size); @@ -59,7 +61,7 @@ class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { // Wrap the data in a stream std::istringstream compiled_stream(data_string); - auto compiled_model = core.import_model(compiled_stream, "CPU"); //target_device); + auto compiled_model = core.import_model(compiled_stream, "CPU"); // Allocate an infer request std::shared_ptr infer_request = std::make_shared(compiled_model.create_infer_request()); @@ -67,7 +69,7 @@ class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { // Allocate execution handle MemoryAllocator* allocator = context.get_runtime_allocator(); ExecutionHandle* handle = ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR(allocator, ExecutionHandle); - handle->compiled_model = std::make_shared(compiled_model); //compiled_model; + handle->compiled_model = std::make_shared(compiled_model); handle->infer_request = infer_request; return handle; @@ -89,14 +91,15 @@ class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { ov::element::Type ov_type = convert_to_openvino_type(input_tensor.scalar_type()); ov::Tensor ov_input_tensor(ov_type, input_shape, input_tensor.mutable_data_ptr()); - infer_request->set_tensor("input", ov_input_tensor); + //infer_request->set_tensor("input", ov_input_tensor); + infer_request->set_input_tensor(0, ov_input_tensor); // Execute the inference infer_request->infer(); // Retrieve and copy output auto output_tensor = args[1]->toTensor(); // Assume second argument is the output - ov::Tensor ov_output_tensor = infer_request->get_tensor("output"); + ov::Tensor ov_output_tensor = infer_request->get_output_tensor(0); //get_tensor("output"); std::memcpy(output_tensor.mutable_data_ptr(), ov_output_tensor.data(), ov_output_tensor.get_byte_size()); @@ -123,12 +126,14 @@ class OpenvinoBackend final : public ::executorch::runtime::BackendInterface { } }; -namespace { -auto backend = OpenvinoBackend(); -Backend backend_id{"OpenvinoBackend", &backend}; -static auto registered = register_backend(backend_id); -} // namespace - } // namespace openvino } // namespace backends } // namespace executorch + +namespace { +auto backend = executorch::backends::openvino::OpenvinoBackend(); +executorch::runtime::Backend backend_id{"OpenvinoBackend", &backend}; +static auto registered = executorch::runtime::register_backend(backend_id); +} // namespace + + diff --git a/examples/openvino/CMakeLists.txt b/examples/openvino/CMakeLists.txt new file mode 100644 index 00000000000..31903042c04 --- /dev/null +++ b/examples/openvino/CMakeLists.txt @@ -0,0 +1,83 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set(CMAKE_CXX_STANDARD 17) + +cmake_minimum_required(VERSION 3.19) +project(openvino_runner_example) + +# Source root directory for executorch. +if(NOT EXECUTORCH_ROOT) + set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) +endif() + +include(${EXECUTORCH_ROOT}/build/Utils.cmake) +include(${EXECUTORCH_ROOT}/build/Codegen.cmake) + +if(NOT PYTHON_EXECUTABLE) + resolve_python_executable() +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() + +set(_common_compile_options -Wno-deprecated-declarations -fPIC) + +# Let files say "include ". +set(_common_include_directories ${EXECUTORCH_ROOT}/..) + +# +# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# +set(EXECUTORCH_SRCS_FILE + "${CMAKE_CURRENT_BINARY_DIR}/../../../build/executorch_srcs.cmake" +) +extract_sources(${EXECUTORCH_SRCS_FILE}) +include(${EXECUTORCH_SRCS_FILE}) + +set(_openvino_executor_runner__srcs ${CMAKE_CURRENT_LIST_DIR}/../openvino/executor_runner/openvino_executor_runner.cpp) + +# preprocess executor runner src files +list(PREPEND _openvino_executor_runner__srcs + ${CMAKE_CURRENT_LIST_DIR}/../openvino/executor_runner/openvino_executor_runner.cpp +) + +# build executor runner +add_executable(openvino_executor_runner ${_openvino_executor_runner__srcs}) +target_include_directories( + openvino_executor_runner PUBLIC ${_common_include_directories} +) + +# Set the path to the library directory +set(LIBRARY_DIR "/home/icx-6338/ynimmaga/executorch_new/executorch/cmake-openvino-out/lib/") + +# List the libraries you want to link (without the 'lib' prefix and file extension) +set(LIBRARIES_TO_LINK ${LIBRARY_DIR}/libopenvino_backend.so + ${LIBRARY_DIR}/libexecutorch.a + ${LIBRARY_DIR}/libexecutorch_core.a + ${EXECUTORCH_ROOT}/third-party/gflags/build/lib/libgflags_nothreads.a + ${LIBRARY_DIR}/libpthreadpool.a + ${LIBRARY_DIR}/libextension_data_loader.a + ${LIBRARY_DIR}/libextension_runner_util.a +) + +# Add the library directory to the link search path +link_directories(${LIBRARY_DIR}) + +# Link all libraries at once +target_link_libraries(openvino_executor_runner PRIVATE ${LIBRARIES_TO_LINK}) + +set_target_properties( + openvino_executor_runner PROPERTIES LINK_FLAGS "-Wl,-rpath='$ORIGIN'" +) + + +get_filename_component( + EXECUTORCH_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE +) + + diff --git a/examples/openvino/executor_runner/openvino_executor_runner.cpp b/examples/openvino/executor_runner/openvino_executor_runner.cpp new file mode 100644 index 00000000000..86b975fe007 --- /dev/null +++ b/examples/openvino/executor_runner/openvino_executor_runner.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +static uint8_t method_allocator_pool[4 * 1024U * 1024U]; // 4 MB + +DEFINE_string( + model_path, + "/home/icx-6338/ynimmaga/delegate.pte", //"model.pte", + "Model serialized in flatbuffer format."); +DEFINE_int32(iteration, 1, "Iterations of inference."); + +using executorch::extension::FileDataLoader; +using executorch::extension::prepare_input_tensors; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; + +int main(int argc, char** argv) { + executorch::runtime::runtime_init(); + + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (argc != 1) { + std::string msg = "Extra commandline args:"; + for (int i = 1; i < argc; i++) { + msg += " " + std::string(argv[i]); + } + ET_LOG(Error, "%s", msg.c_str()); + return 1; + } + + const char* model_path = FLAGS_model_path.c_str(); + Result loader = FileDataLoader::from(model_path); + ET_CHECK_MSG( + loader.ok(), + "FileDataLoader::from() failed: 0x%" PRIx32, + static_cast(loader.error())); + + Result program = Program::load(&loader.get()); + if (!program.ok()) { + ET_LOG(Error, "Failed to parse model file %s", model_path); + return 1; + } + ET_LOG(Info, "Model file %s is loaded.", model_path); + + const char* method_name = nullptr; + { + const auto method_name_result = program->get_method_name(0); + ET_CHECK_MSG(method_name_result.ok(), "Program has no methods"); + method_name = *method_name_result; + } + ET_LOG(Info, "Using method %s", method_name); + + Result method_meta = program->method_meta(method_name); + ET_CHECK_MSG( + method_meta.ok(), + "Failed to get method_meta for %s: 0x%" PRIx32, + method_name, + static_cast(method_meta.error())); + + MemoryAllocator method_allocator{ + MemoryAllocator(sizeof(method_allocator_pool), method_allocator_pool)}; + + std::vector> planned_buffers; + std::vector> planned_spans; + size_t num_memory_planned_buffers = method_meta->num_memory_planned_buffers(); + for (size_t id = 0; id < num_memory_planned_buffers; ++id) { + size_t buffer_size = + static_cast(method_meta->memory_planned_buffer_size(id).get()); + ET_LOG(Info, "Setting up planned buffer %zu, size %zu.", id, buffer_size); + planned_buffers.push_back(std::make_unique(buffer_size)); + planned_spans.push_back({planned_buffers.back().get(), buffer_size}); + } + HierarchicalAllocator planned_memory( + {planned_spans.data(), planned_spans.size()}); + + MemoryManager memory_manager(&method_allocator, &planned_memory); + + Result method = program->load_method(method_name, &memory_manager); + ET_CHECK_MSG( + method.ok(), + "Loading of method %s failed with status 0x%" PRIx32, + method_name, + static_cast(method.error())); + ET_LOG(Info, "Method loaded."); + + auto inputs = prepare_input_tensors(*method); + ET_CHECK_MSG( + inputs.ok(), + "Could not prepare inputs: 0x%" PRIx32, + static_cast(inputs.error())); + ET_LOG(Info, "Inputs prepared."); + + auto before_exec = std::chrono::high_resolution_clock::now(); + Error status = Error::Ok; + for (int i = 0; i < FLAGS_iteration; ++i) { + status = method->execute(); + } + auto after_exec = std::chrono::high_resolution_clock::now(); + double elapsed_time = std::chrono::duration_cast( + after_exec - before_exec) + .count() / 1000.0; + + ET_LOG( + Info, + "%d inference took %f ms, avg %f ms", + FLAGS_iteration, + elapsed_time, + elapsed_time / static_cast(FLAGS_iteration)); + ET_CHECK_MSG( + status == Error::Ok, + "Execution of method %s failed with status 0x%" PRIx32, + method_name, + static_cast(status)); + ET_LOG(Info, "Model executed successfully."); + + std::vector outputs(method->outputs_size()); + ET_LOG(Info, "%zu outputs: ", outputs.size()); + status = method->get_outputs(outputs.data(), outputs.size()); + ET_CHECK(status == Error::Ok); + //std::cout << executorch::extension::evalue_edge_items(100); + //for (int i = 0; i < outputs.size(); ++i) { + // std::cout << "Output " << i << ": " << outputs[i] << std::endl; + //} + + return 0; +} diff --git a/examples/openvino/executor_runner/ov_executor_runner.cpp b/examples/openvino/executor_runner/ov_executor_runner.cpp new file mode 100644 index 00000000000..d0be48fdcc9 --- /dev/null +++ b/examples/openvino/executor_runner/ov_executor_runner.cpp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +/* +using executorch::extension::FileDataLoader; +using executorch::extension::prepare_input_tensors; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; +*/ +using executorch::aten::Tensor; +using executorch::aten::TensorImpl; +using executorch::extension::FileDataLoader; +//using executorch::extension::MallocMemoryAllocator; +using executorch::extension::prepare_input_tensors; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; + +int main() { +Result loader = + FileDataLoader::from("/home/icx-6338/ynimmaga/delegate.pte"); +assert(loader.ok()); + +Result program = Program::load(&loader.get()); +assert(program.ok()); + +// Method names map back to Python nn.Module method names. Most users will only +// have the singular method "forward". +const char* method_name = "forward"; + +// MethodMeta is a lightweight structure that lets us gather metadata +// information about a specific method. In this case we are looking to get the +// required size of the memory planned buffers for the method "forward". +Result method_meta = program->method_meta(method_name); +assert(method_meta.ok()); + +std::vector> planned_buffers; // Owns the Memory +std::vector> planned_arenas; // Passed to the allocator + +size_t num_memory_planned_buffers = method_meta->num_memory_planned_buffers(); + +// It is possible to have multiple layers in our memory hierarchy; for example, +// SRAM and DRAM. +for (size_t id = 0; id < num_memory_planned_buffers; ++id) { + // .get() will always succeed because id < num_memory_planned_buffers. + size_t buffer_size = + static_cast(method_meta->memory_planned_buffer_size(id).get()); + planned_buffers.push_back(std::make_unique(buffer_size)); + planned_arenas.push_back({planned_buffers.back().get(), buffer_size}); +} + +HierarchicalAllocator planned_memory( + {planned_arenas.data(), planned_arenas.size()}); + +// Version of MemoryAllocator that uses malloc to handle allocations rather then +// a fixed buffer. +//MallocMemoryAllocator method_allocator; +MemoryAllocator method_allocator{ + MemoryAllocator(sizeof(method_allocator_pool), method_allocator_pool)}; + +// Assemble all of the allocators into the MemoryManager that the Executor will +// use. +MemoryManager memory_manager(&method_allocator, &planned_memory); + +Result method = program->load_method(method_name); +assert(method.ok()); + +// Create our input tensor. +float data[1 * 3 * 256 * 256]; +Tensor::SizesType sizes[] = {1, 3, 256, 256}; +Tensor::DimOrderType dim_order = {0, 1, 2, 3}; +TensorImpl impl( + ScalarType::Float, // dtype + 4, // number of dimensions + sizes, + data, + dim_order); +Tensor t(&impl); + +// Implicitly casts t to EValue +Error set_input_error = method->set_input(t, 0); +assert(set_input_error == Error::Ok); + +Error execute_error = method->execute(); +assert(execute_error == Error::Ok); + +EValue output = method->get_output(0); +assert(output.isTensor()); + +return 0; + +} diff --git a/examples/openvino/openvino_build.sh b/examples/openvino/openvino_build.sh new file mode 100755 index 00000000000..f53679cc910 --- /dev/null +++ b/examples/openvino/openvino_build.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +# Define the directory where CMakeLists.txt is located +EXECUTORCH_ROOT=$(realpath "$(dirname "$0")/../..") +echo EXECUTORCH_ROOT=${EXECUTORCH_ROOT} + +main() { + # Set build directory + local build_dir="cmake-openvino-out" + + # Create and enter the build directory + cd "$EXECUTORCH_ROOT" + rm -rf "${build_dir}" + + # Configure the project with CMake + # Note: Add any additional configuration options you need here + cmake -DCMAKE_INSTALL_PREFIX="${build_dir}" \ + -DEXECUTORCH_BUILD_OPENVINO=ON \ + -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ + -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ + -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \ + -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ + -B"${build_dir}" + + + # Build the project + cmake --build cmake-openvino-out --target install --config Release -j5 + + ## Build example + local example_dir=examples/openvino + local example_build_dir="${build_dir}/${example_dir}" + local cmake_prefix_path="${PWD}/${build_dir}/lib/cmake/ExecuTorch;${PWD}/${build_dir}/third-party/gflags;" + rm -rf "${example_build_dir}" + + ## MTK original + cmake -DCMAKE_PREFIX_PATH="${cmake_prefix_path}" \ + -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \ + -B"${example_build_dir}" \ + $EXECUTORCH_ROOT/$example_dir + + cmake --build "${example_build_dir}" -j5 + + # Switch back to the original directory + cd - > /dev/null + + # Print a success message + echo "Build successfully completed." +} + +main "$@"