diff --git a/cmake/deps.txt b/cmake/deps.txt index d1a528bd6b4be..482f2496cf3fb 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,3 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b +vaip;https://github.com/amd/vaip/archive/refs/tags/v1.0.0-rc0.zip;e084b71c0277e2e3263179a4682300d427ff83e7 diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake index 561a323533f48..a664ecb0e3b75 100644 --- a/cmake/onnxruntime_providers_vitisai.cmake +++ b/cmake/onnxruntime_providers_vitisai.cmake @@ -1,5 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. + FetchContent_Declare( + vaip + URL ${DEP_URL_vaip} + URL_HASH SHA1=${DEP_SHA1_vaip} + OVERRIDE_FIND_PACKAGE + ) + find_package(vaip) if ("${GIT_COMMIT_ID}" STREQUAL "") execute_process( @@ -21,10 +28,11 @@ source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs}) onnxruntime_add_shared_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_vitisai ${ONNXRUNTIME_PROVIDERS_SHARED} ${GSL_TARGET} safeint_interface flatbuffers::flatbuffers) - target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED}) + target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED} onnxruntime_vitisai_ep::onnxruntime_vitisai_ep) if(MSVC) onnxruntime_add_include_to_target(onnxruntime_providers_vitisai dbghelp) set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/vitisai/symbols.def") + target_sources(onnxruntime_providers_vitisai PRIVATE ${vaip_BINARY_DIR}/onnxruntime_vitisai_ep/onnxruntime_vitisai_ep.def) else(MSVC) set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/vitisai/version_script.lds -Xlinker --gc-sections") endif(MSVC) @@ -43,6 +51,10 @@ target_compile_options(onnxruntime_providers_vitisai PRIVATE -Wno-unused-parameter) endif(MSVC) + if(MSVC) + target_link_options(onnxruntime_providers_vitisai PRIVATE "/NODEFAULTLIB:libucrt.lib" "/DEFAULTLIB:ucrt.lib") + endif(MSVC) + set_target_properties(onnxruntime_providers_vitisai PROPERTIES FOLDER "ONNXRuntime") set_target_properties(onnxruntime_providers_vitisai PROPERTIES LINKER_LANGUAGE CXX) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index 8111ee3c1fe61..fd573b9340774 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -41,7 +41,27 @@ using namespace onnxruntime; #define LIBRARY_PREFIX "lib" #define LIBRARY_EXTENSION ".so" #endif - +extern "C" { +void initialize_onnxruntime_vitisai_ep_c(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); +uint32_t vaip_get_version_c(); +int create_ep_context_nodes_c( + const std::vector>& eps, + vaip_core::DllSafe>* ret_value); +std::vector>* compile_onnx_model_with_options_c( + const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); +std::vector>* compile_onnx_model_vitisai_ep_with_error_handling_c( + const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func); +int vitisai_ep_on_run_start_c( + const std::vector>& eps, const void* state, + vaip_core::DllSafe (*get_config_entry)(const void* state, const char* entry_name)); +int vitisai_ep_set_ep_dynamic_options_c( + const std::vector>& eps, + const char* const* keys, + const char* const* values, size_t kv_len); +void profiler_collect_c( + std::vector& api_events, + std::vector& kernel_events); +}; vaip_core::OrtApiForVaip* create_org_api_hook(); struct OrtVitisAIEpAPI { void (*initialize_onnxruntime_vitisai_ep)(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); @@ -50,7 +70,7 @@ struct OrtVitisAIEpAPI { std::vector>* (*compile_onnx_model_vitisai_ep_with_error_handling)( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func); uint32_t (*vaip_get_version)(); - void (*create_ep_context_nodes)( + int (*create_ep_context_nodes)( const std::vector>& eps, vaip_core::DllSafe>* ret_value) = nullptr; int (*vitisai_ep_on_run_start)( @@ -63,34 +83,54 @@ struct OrtVitisAIEpAPI { void (*profiler_collect)( std::vector& api_events, std::vector& kernel_events); + int (*vaip_xcompiler_compile)(const char* input_xmodel, + size_t input_xmodel_size, + const char* config_xmodel, + size_t config_xmodel_size, void* state, + void (*k)(void*, void*, size_t)) = nullptr; + const char* (*vaip_get_default_config)() = nullptr; + int (*vaip_get_pattern_as_binary)(const char* name, void* state, void (*k)(void*, void*, size_t)) = nullptr; + void (*vaip_get_pattern_list)(void* state, void (*k)(void*, void*, size_t)) = nullptr; + int (*vaip_get_mem_xclbin)(const char* name, void* state, void (*k)(void*, void*, size_t)) = nullptr; + bool (*vaip_has_mem_xclbin)(const char* name) = nullptr; void Ensure() { if (handle_) return; + + this->initialize_onnxruntime_vitisai_ep = initialize_onnxruntime_vitisai_ep_c; + this->compile_onnx_model_with_options = compile_onnx_model_with_options_c; + this->compile_onnx_model_vitisai_ep_with_error_handling = compile_onnx_model_vitisai_ep_with_error_handling_c; + this->create_ep_context_nodes = create_ep_context_nodes_c; + this->vitisai_ep_on_run_start = vitisai_ep_on_run_start_c; + this->vitisai_ep_set_ep_dynamic_options = vitisai_ep_set_ep_dynamic_options_c; + this->vaip_get_version = vaip_get_version_c; + this->profiler_collect = profiler_collect_c; + auto& env = Provider_GetHost()->Env__Default(); + auto& logger = *Provider_GetHost()->LoggingManager_GetDefaultLogger(); + #ifdef _WIN32 // this dll is already linked to the executable, normally a test program handle_ = reinterpret_cast(GetModuleHandle(TEXT("onnxruntime_vitisai_ep.dll"))); + auto status = Status::OK(); if (!handle_) { auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION); - ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); + status = env.LoadDynamicLibrary(full_path, true, &handle_); + if (!status.IsOK()) { + LOGS(logger, VERBOSE) << "cannot load onnxruntime_vitisai_ep.dll, can only deploy ep_context onnx model."; + return; + } } #else auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION); ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); #endif - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "initialize_onnxruntime_vitisai_ep", (void**)&initialize_onnxruntime_vitisai_ep)); - auto status1 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_error_handling", (void**)&compile_onnx_model_vitisai_ep_with_error_handling); - auto status2 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_options", (void**)&compile_onnx_model_with_options); - if ((!status1.IsOK()) && (!status2.IsOK())) { - ::onnxruntime::LogRuntimeError(0, status2, __FILE__, static_cast(__FUNCTION__), __LINE__); - ORT_THROW(status2); - } - std::ignore = env.GetSymbolFromLibrary(handle_, "vaip_get_version", - (void**)&vaip_get_version); - std::ignore = env.GetSymbolFromLibrary(handle_, "profiler_collect", (void**)&profiler_collect); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "create_ep_context_nodes", (void**)&create_ep_context_nodes)); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_on_run_start", (void**)&vitisai_ep_on_run_start)); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_set_ep_dynamic_options", (void**)&vitisai_ep_set_ep_dynamic_options)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_xcompiler_compile", (void**)&vaip_xcompiler_compile)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_default_config", (void**)&vaip_get_default_config)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_pattern_as_binary", (void**)&vaip_get_pattern_as_binary)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_pattern_list", (void**)&vaip_get_pattern_list)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_mem_xclbin", (void**)&vaip_get_mem_xclbin)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_has_mem_xclbin", (void**)&vaip_has_mem_xclbin)); } void Clear() { if (handle_) { @@ -484,6 +524,7 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { } }; the_global_api.node_arg_external_location = vaip::node_arg_external_location; + the_global_api.model_to_proto = [](onnxruntime::Model& model) { return model.ToProto().release(); }; the_global_api.model_proto_serialize_as_string = [](ONNX_NAMESPACE::ModelProto& model_proto) { return vaip_core::DllSafe(model_proto.SerializeAsString()); @@ -503,6 +544,12 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { the_global_api.graph_remove_initialized_tensor = [](Graph& graph, const std::string& tensor_name) { graph.RemoveInitializedTensor(tensor_name); }; + the_global_api.vaip_xcompiler_compile = s_library_vitisaiep.vaip_xcompiler_compile; + the_global_api.vaip_get_default_config = s_library_vitisaiep.vaip_get_default_config; + the_global_api.vaip_get_pattern_as_binary = s_library_vitisaiep.vaip_get_pattern_as_binary; + the_global_api.vaip_get_pattern_list = s_library_vitisaiep.vaip_get_pattern_list; + the_global_api.vaip_get_mem_xclbin = s_library_vitisaiep.vaip_get_mem_xclbin; + the_global_api.vaip_has_mem_xclbin = s_library_vitisaiep.vaip_has_mem_xclbin; if (!s_library_vitisaiep.vaip_get_version) { return reinterpret_cast(&(the_global_api.host_)); } else { diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 6a51ef862280b..9d999040c885e 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -235,7 +235,7 @@ struct OrtApiForVaip { DllSafe (*model_proto_serialize_as_string)(ModelProto& model_proto); // [96] void (*model_proto_delete)(ModelProto* p); // [97] DllSafe (*attr_proto_release_string)(AttributeProto* attr); // [98] - bool (*is_profiling_enabled)(void* session_options); // [99] // [98] + bool (*is_profiling_enabled)(void* session_options); // [99] TensorProto* (*tensor_proto_new_i4)(const std::string& name, const std::vector& shape, const std::vector& data); // [100] @@ -243,6 +243,16 @@ struct OrtApiForVaip { const std::vector& shape, const std::vector& data); // [101] void (*graph_remove_initialized_tensor)(Graph& graph, const std::string& tensor_name); // [102] + int (*vaip_xcompiler_compile)(const char* input_xmodel, + size_t input_xmodel_size, + const char* config_xmodel, + size_t config_xmodel_size, void* state, + void (*k)(void*, void*, size_t)); // [103] + const char* (*vaip_get_default_config)(); // [104] + int (*vaip_get_pattern_as_binary)(const char* name, void* state, void (*k)(void*, void*, size_t)); // [105] + void (*vaip_get_pattern_list)(void* state, void (*k)(void*, void*, size_t)); // [106] + int (*vaip_get_mem_xclbin)(const char* name, void* state, void (*k)(void*, void*, size_t)); // [107] + bool (*vaip_has_mem_xclbin)(const char* name); // [108] }; #ifndef USE_VITISAI