From 7cbcf5cd4b12fd264083a250409227e65961817f Mon Sep 17 00:00:00 2001 From: Chunye Wang Date: Mon, 19 Aug 2024 23:21:01 -0700 Subject: [PATCH 01/10] refactor VitisAI EP for opensource update to use integrated vaip update vaip as the single entry point for cmake add dummy vaip_xcompile_run. onnxruntime_vitisia_ep.dll is optional. vaip_xcompiler_compile maybe nullptr. update create_ep_context_nodes. --- cmake/deps.txt | 1 + .../external/onnxruntime_external_deps.cmake | 1 - cmake/onnxruntime_providers_vitisai.cmake | 24 +++++++- .../core/providers/vitisai/imp/global_api.cc | 55 ++++++++++++------- .../vitisai/include/vaip/vaip_ort_api.h | 7 ++- 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index d1a528bd6b4be..2eb78d57566cb 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,3 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b +vaip;https://gitenterprise.xilinx.com/chunywan/vaip.git;refactor-vaip diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 761ce47582ee5..773a5ee46051f 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -198,7 +198,6 @@ FetchContent_Declare( PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND} FIND_PACKAGE_ARGS NAMES Protobuf protobuf ) - set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #TODO: we'd better to turn the following option off. However, it will cause # ".\build.bat --config Debug --parallel --skip_submodule_sync --update" fail with an error message: diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake index 561a323533f48..fa87cc4be39f9 100644 --- a/cmake/onnxruntime_providers_vitisai.cmake +++ b/cmake/onnxruntime_providers_vitisai.cmake @@ -1,5 +1,26 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. + FetchContent_Declare( + vaip + GIT_REPOSITORY ${DEP_URL_vaip} + GIT_TAG ${DEP_SHA1_vaip} + GIT_SUBMODULES_RECURSE FALSE + GIT_SHALLOW TRUE + OVERRIDE_FIND_PACKAGE + ) + set(WITH_XCOMPILER ON CACHE BOOL "enable XCOMPILER") + set(WITH_OPENSSL OFF CACHE BOOL "enable open ssl") + set(WITH_CPURUNNER ON CACHE BOOL "enable cpu runner") + set(BUILD_PYTHON_EXT OFF CACHE BOOL "enable python ext") + set(EN_LLM_DOD_OPS OFF CACHE BOOL "enable dd flow") + set(EN_VAIML OFF CACHE BOOL "enable vaiml flow") + set(TRIM_CONFIG OFF CACHE BOOL "enable config trimming") + set(ENABLE_VITIS_AI_CUSTOM_OP OFF "enable vitis ai custom op") + set(PACK_XCLBIN_PATH "" CACHE STRING "list of xclbin files") + set(ENABLE_BUILD_VOE_WHEEL OFF CACHE BOOL "internal used only" FORCE) + set(INSTALL_USER ON CACHE BOOL "internal used only" FORCE) + set(ENABLE_XRT_SHARED_CONTEXT ON CACHE BOOL "internal used only" FORCE) + find_package(vaip) if ("${GIT_COMMIT_ID}" STREQUAL "") execute_process( @@ -21,10 +42,11 @@ source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs}) onnxruntime_add_shared_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_vitisai ${ONNXRUNTIME_PROVIDERS_SHARED} ${GSL_TARGET} safeint_interface flatbuffers::flatbuffers) - target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED}) + target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED} onnxruntime_vitisai_ep::onnxruntime_vitisai_ep) if(MSVC) onnxruntime_add_include_to_target(onnxruntime_providers_vitisai dbghelp) set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/vitisai/symbols.def") + target_sources(onnxruntime_providers_vitisai PRIVATE ${vaip_BINARY_DIR}/onnxruntime_vitisai_ep/onnxruntime_vitisai_ep.def) else(MSVC) set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/vitisai/version_script.lds -Xlinker --gc-sections") endif(MSVC) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index 8111ee3c1fe61..c017ce922d46d 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -42,6 +42,16 @@ using namespace onnxruntime; #define LIBRARY_EXTENSION ".so" #endif +namespace vaip_core { + +void initialize_onnxruntime_vitisai_ep(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); +uint32_t vaip_get_version(); +extern "C" int create_ep_context_nodes( + const std::vector>& eps, + vaip_core::DllSafe>* ret_value); +std::vector>* compile_onnx_model_with_options( + const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); +} // namespace vaip_core vaip_core::OrtApiForVaip* create_org_api_hook(); struct OrtVitisAIEpAPI { void (*initialize_onnxruntime_vitisai_ep)(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); @@ -50,7 +60,7 @@ struct OrtVitisAIEpAPI { std::vector>* (*compile_onnx_model_vitisai_ep_with_error_handling)( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func); uint32_t (*vaip_get_version)(); - void (*create_ep_context_nodes)( + int (*create_ep_context_nodes)( const std::vector>& eps, vaip_core::DllSafe>* ret_value) = nullptr; int (*vitisai_ep_on_run_start)( @@ -63,34 +73,37 @@ struct OrtVitisAIEpAPI { void (*profiler_collect)( std::vector& api_events, std::vector& kernel_events); + int (*vaip_xcompiler_compile)(const char* input_xmodel, + size_t input_xmodel_size, + const char* config_xmodel, + size_t config_xmodel_size, void* state, + char* (*allocator)(void*, size_t)) = nullptr; void Ensure() { - if (handle_) - return; + this->initialize_onnxruntime_vitisai_ep = vaip_core::initialize_onnxruntime_vitisai_ep; + this->compile_onnx_model_with_options = vaip_core::compile_onnx_model_with_options; + this->create_ep_context_nodes = vaip_core::create_ep_context_nodes; + this->vaip_get_version = vaip_core::vaip_get_version; + // auto& env = Provider_GetHost()->Env__Default(); + auto& logger = *Provider_GetHost()->LoggingManager_GetDefaultLogger(); + #ifdef _WIN32 // this dll is already linked to the executable, normally a test program handle_ = reinterpret_cast(GetModuleHandle(TEXT("onnxruntime_vitisai_ep.dll"))); + auto status = Status::OK(); if (!handle_) { auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION); - ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); + status = env.LoadDynamicLibrary(full_path, true, &handle_); + if (!status.IsOK()) { + LOGS(logger, VERBOSE) << "cannot load onnxruntime_vitisai_ep.dll, can only deploy ep_context onnx model."; + return; + } } #else auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION); ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); #endif - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "initialize_onnxruntime_vitisai_ep", (void**)&initialize_onnxruntime_vitisai_ep)); - auto status1 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_error_handling", (void**)&compile_onnx_model_vitisai_ep_with_error_handling); - auto status2 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_options", (void**)&compile_onnx_model_with_options); - if ((!status1.IsOK()) && (!status2.IsOK())) { - ::onnxruntime::LogRuntimeError(0, status2, __FILE__, static_cast(__FUNCTION__), __LINE__); - ORT_THROW(status2); - } - std::ignore = env.GetSymbolFromLibrary(handle_, "vaip_get_version", - (void**)&vaip_get_version); - std::ignore = env.GetSymbolFromLibrary(handle_, "profiler_collect", (void**)&profiler_collect); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "create_ep_context_nodes", (void**)&create_ep_context_nodes)); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_on_run_start", (void**)&vitisai_ep_on_run_start)); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_set_ep_dynamic_options", (void**)&vitisai_ep_set_ep_dynamic_options)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_xcompiler_compile", (void**)&vaip_xcompiler_compile)); } void Clear() { if (handle_) { @@ -107,9 +120,10 @@ struct OrtVitisAIEpAPI { static OrtVitisAIEpAPI s_library_vitisaiep; static std::shared_ptr s_kernel_registry_vitisaiep; -static std::vector s_domains_vitisaiep; static vaip_core::OrtApiForVaip the_global_api; -std::shared_ptr get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } +g + std::shared_ptr + get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } const std::vector& get_domains_vitisaiep() { return s_domains_vitisaiep; } void profiler_collect( @@ -484,6 +498,7 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { } }; the_global_api.node_arg_external_location = vaip::node_arg_external_location; + the_global_api.model_to_proto = [](onnxruntime::Model& model) { return model.ToProto().release(); }; the_global_api.model_proto_serialize_as_string = [](ONNX_NAMESPACE::ModelProto& model_proto) { return vaip_core::DllSafe(model_proto.SerializeAsString()); @@ -508,6 +523,8 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { } else { return &the_global_api; } + the_global_api.vaip_xcompiler_compile = s_library_vitisaiep.vaip_xcompiler_compile; + return &the_global_api; } void deinitialize_vitisai_ep() { diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 6a51ef862280b..2494558b1a778 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -235,7 +235,7 @@ struct OrtApiForVaip { DllSafe (*model_proto_serialize_as_string)(ModelProto& model_proto); // [96] void (*model_proto_delete)(ModelProto* p); // [97] DllSafe (*attr_proto_release_string)(AttributeProto* attr); // [98] - bool (*is_profiling_enabled)(void* session_options); // [99] // [98] + bool (*is_profiling_enabled)(void* session_options); // [99] TensorProto* (*tensor_proto_new_i4)(const std::string& name, const std::vector& shape, const std::vector& data); // [100] @@ -243,6 +243,11 @@ struct OrtApiForVaip { const std::vector& shape, const std::vector& data); // [101] void (*graph_remove_initialized_tensor)(Graph& graph, const std::string& tensor_name); // [102] + int (*vaip_xcompiler_compile)(const char* input_xmodel, + size_t input_xmodel_size, + const char* config_xmodel, + size_t config_xmodel_size, void* state, + char* (*allocator)(void*, size_t)); // [103] }; #ifndef USE_VITISAI From 6eeafca6e0ba186a64b13e527f541d5518671f33 Mon Sep 17 00:00:00 2001 From: Liu Minyue Date: Mon, 26 Aug 2024 03:28:08 -0600 Subject: [PATCH 02/10] Add some VitisAI EP C APIs fix compile error move configuration to vaip update vaip repo remote url add vaip_get_default_config API move pattern zoo to internal (#1) * move pattern zoo to internal * change external vaip --------- Co-authored-by: mingyue Co-authored-by: mingyue add get_patterrn_list API and change get_pattern API (#2) * add get_patterrn_list API and change get_pattern API * lintrunner -a --------- Co-authored-by: mingyue Co-authored-by: mingyue change xcompiler_compile API by CPS (#3) Co-authored-by: mingyue [deps] change vaip remote and branch Add vaip_get/has_mem_xclbin APIs (#4) * Add vaip_get/has_mem_xclbin APIs --------- Co-authored-by: mingyue vaip point to github/amd use main branch --- cmake/deps.txt | 2 +- .../external/onnxruntime_external_deps.cmake | 1 + cmake/onnxruntime_providers_vitisai.cmake | 12 ---- .../core/providers/vitisai/imp/global_api.cc | 55 +++++++++++++------ .../vitisai/include/vaip/vaip_ort_api.h | 7 ++- 5 files changed, 45 insertions(+), 32 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index 2eb78d57566cb..d5ba9c48fe46e 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,4 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b -vaip;https://gitenterprise.xilinx.com/chunywan/vaip.git;refactor-vaip +vaip;https://github.com/amd/vaip.git;main diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 773a5ee46051f..761ce47582ee5 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -198,6 +198,7 @@ FetchContent_Declare( PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND} FIND_PACKAGE_ARGS NAMES Protobuf protobuf ) + set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #TODO: we'd better to turn the following option off. However, it will cause # ".\build.bat --config Debug --parallel --skip_submodule_sync --update" fail with an error message: diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake index fa87cc4be39f9..3d7f631e42f85 100644 --- a/cmake/onnxruntime_providers_vitisai.cmake +++ b/cmake/onnxruntime_providers_vitisai.cmake @@ -8,18 +8,6 @@ GIT_SHALLOW TRUE OVERRIDE_FIND_PACKAGE ) - set(WITH_XCOMPILER ON CACHE BOOL "enable XCOMPILER") - set(WITH_OPENSSL OFF CACHE BOOL "enable open ssl") - set(WITH_CPURUNNER ON CACHE BOOL "enable cpu runner") - set(BUILD_PYTHON_EXT OFF CACHE BOOL "enable python ext") - set(EN_LLM_DOD_OPS OFF CACHE BOOL "enable dd flow") - set(EN_VAIML OFF CACHE BOOL "enable vaiml flow") - set(TRIM_CONFIG OFF CACHE BOOL "enable config trimming") - set(ENABLE_VITIS_AI_CUSTOM_OP OFF "enable vitis ai custom op") - set(PACK_XCLBIN_PATH "" CACHE STRING "list of xclbin files") - set(ENABLE_BUILD_VOE_WHEEL OFF CACHE BOOL "internal used only" FORCE) - set(INSTALL_USER ON CACHE BOOL "internal used only" FORCE) - set(ENABLE_XRT_SHARED_CONTEXT ON CACHE BOOL "internal used only" FORCE) find_package(vaip) if ("${GIT_COMMIT_ID}" STREQUAL "") diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index c017ce922d46d..bb9e306bcd080 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -41,17 +41,15 @@ using namespace onnxruntime; #define LIBRARY_PREFIX "lib" #define LIBRARY_EXTENSION ".so" #endif - -namespace vaip_core { - -void initialize_onnxruntime_vitisai_ep(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); -uint32_t vaip_get_version(); -extern "C" int create_ep_context_nodes( +extern "C" { +void initialize_onnxruntime_vitisai_ep_c(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); +uint32_t vaip_get_version_c(); +int create_ep_context_nodes_c( const std::vector>& eps, vaip_core::DllSafe>* ret_value); -std::vector>* compile_onnx_model_with_options( +std::vector>* compile_onnx_model_with_options_c( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); -} // namespace vaip_core +}; vaip_core::OrtApiForVaip* create_org_api_hook(); struct OrtVitisAIEpAPI { void (*initialize_onnxruntime_vitisai_ep)(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); @@ -77,13 +75,21 @@ struct OrtVitisAIEpAPI { size_t input_xmodel_size, const char* config_xmodel, size_t config_xmodel_size, void* state, - char* (*allocator)(void*, size_t)) = nullptr; + void (*k)(void*, void*, size_t)) = nullptr; + const char* (*vaip_get_default_config)() = nullptr; + int (*vaip_get_pattern_as_binary)(const char* name, void* state, void (*k)(void*, void*, size_t)) = nullptr; + void (*vaip_get_pattern_list)(void* state, void (*k)(void*, void*, size_t)) = nullptr; + int (*vaip_get_mem_xclbin)(const char* name, void* state, void (*k)(void*, void*, size_t)) = nullptr; + bool (*vaip_has_mem_xclbin)(const char* name) = nullptr; void Ensure() { - this->initialize_onnxruntime_vitisai_ep = vaip_core::initialize_onnxruntime_vitisai_ep; - this->compile_onnx_model_with_options = vaip_core::compile_onnx_model_with_options; - this->create_ep_context_nodes = vaip_core::create_ep_context_nodes; - this->vaip_get_version = vaip_core::vaip_get_version; - // + if (handle_) + return; + + this->initialize_onnxruntime_vitisai_ep = initialize_onnxruntime_vitisai_ep_c; + this->compile_onnx_model_with_options = compile_onnx_model_with_options_c; + this->create_ep_context_nodes = create_ep_context_nodes_c; + this->vaip_get_version = vaip_get_version_c; + auto& env = Provider_GetHost()->Env__Default(); auto& logger = *Provider_GetHost()->LoggingManager_GetDefaultLogger(); @@ -104,6 +110,11 @@ struct OrtVitisAIEpAPI { ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); #endif ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_xcompiler_compile", (void**)&vaip_xcompiler_compile)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_default_config", (void**)&vaip_get_default_config)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_pattern_as_binary", (void**)&vaip_get_pattern_as_binary)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_pattern_list", (void**)&vaip_get_pattern_list)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_mem_xclbin", (void**)&vaip_get_mem_xclbin)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_has_mem_xclbin", (void**)&vaip_has_mem_xclbin)); } void Clear() { if (handle_) { @@ -120,10 +131,9 @@ struct OrtVitisAIEpAPI { static OrtVitisAIEpAPI s_library_vitisaiep; static std::shared_ptr s_kernel_registry_vitisaiep; +static std::vector s_domains_vitisaiep; static vaip_core::OrtApiForVaip the_global_api; -g - std::shared_ptr - get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } +std::shared_ptr get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } const std::vector& get_domains_vitisaiep() { return s_domains_vitisaiep; } void profiler_collect( @@ -524,7 +534,16 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { return &the_global_api; } the_global_api.vaip_xcompiler_compile = s_library_vitisaiep.vaip_xcompiler_compile; - return &the_global_api; + the_global_api.vaip_get_default_config = s_library_vitisaiep.vaip_get_default_config; + the_global_api.vaip_get_pattern_as_binary = s_library_vitisaiep.vaip_get_pattern_as_binary; + the_global_api.vaip_get_pattern_list = s_library_vitisaiep.vaip_get_pattern_list; + the_global_api.vaip_get_mem_xclbin = s_library_vitisaiep.vaip_get_mem_xclbin; + the_global_api.vaip_has_mem_xclbin = s_library_vitisaiep.vaip_has_mem_xclbin; + if (!s_library_vitisaiep.vaip_get_version) { + return reinterpret_cast(&(the_global_api.host_)); + } else { + return &the_global_api; + } } void deinitialize_vitisai_ep() { diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 2494558b1a778..9d999040c885e 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -247,7 +247,12 @@ struct OrtApiForVaip { size_t input_xmodel_size, const char* config_xmodel, size_t config_xmodel_size, void* state, - char* (*allocator)(void*, size_t)); // [103] + void (*k)(void*, void*, size_t)); // [103] + const char* (*vaip_get_default_config)(); // [104] + int (*vaip_get_pattern_as_binary)(const char* name, void* state, void (*k)(void*, void*, size_t)); // [105] + void (*vaip_get_pattern_list)(void* state, void (*k)(void*, void*, size_t)); // [106] + int (*vaip_get_mem_xclbin)(const char* name, void* state, void (*k)(void*, void*, size_t)); // [107] + bool (*vaip_has_mem_xclbin)(const char* name); // [108] }; #ifndef USE_VITISAI From def49fe12f396417db01940a34fddb7ed638be9a Mon Sep 17 00:00:00 2001 From: mingyue Date: Wed, 6 Nov 2024 19:54:42 -0700 Subject: [PATCH 03/10] fix rebase error --- onnxruntime/core/providers/vitisai/imp/global_api.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index bb9e306bcd080..b1eeb49dfd8dc 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -49,6 +49,13 @@ int create_ep_context_nodes_c( vaip_core::DllSafe>* ret_value); std::vector>* compile_onnx_model_with_options_c( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); +int vitisai_ep_on_run_start_c( + const std::vector>& eps, const void* state, + vaip_core::DllSafe (*get_config_entry)(const void* state, const char* entry_name)); +int vitisai_ep_set_ep_dynamic_options_c( + const std::vector>& eps, + const char* const* keys, + const char* const* values, size_t kv_len); }; vaip_core::OrtApiForVaip* create_org_api_hook(); struct OrtVitisAIEpAPI { @@ -88,6 +95,8 @@ struct OrtVitisAIEpAPI { this->initialize_onnxruntime_vitisai_ep = initialize_onnxruntime_vitisai_ep_c; this->compile_onnx_model_with_options = compile_onnx_model_with_options_c; this->create_ep_context_nodes = create_ep_context_nodes_c; + this->vitisai_ep_on_run_start = vitisai_ep_on_run_start_c; + this->vitisai_ep_set_ep_dynamic_options = vitisai_ep_set_ep_dynamic_options_c; this->vaip_get_version = vaip_get_version_c; auto& env = Provider_GetHost()->Env__Default(); From 20e4374d58b949816e99fc371bca2d475612c474 Mon Sep 17 00:00:00 2001 From: mingyue Date: Thu, 14 Nov 2024 01:05:24 -0700 Subject: [PATCH 04/10] use internal vaip for rebase test --- cmake/deps.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index d5ba9c48fe46e..e8e0cd03bf5dd 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,4 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b -vaip;https://github.com/amd/vaip.git;main +vaip;https://gitenterprise.xilinx.com/VitisAI/vaip.git;win24_open_src From dcd9931533cc856fb1b2980126c2afc55a9a98e5 Mon Sep 17 00:00:00 2001 From: mingyue Date: Fri, 15 Nov 2024 02:59:58 -0700 Subject: [PATCH 05/10] onnxruntime_providers_vitisai.dll enable shared link Hybrid CRT --- cmake/onnxruntime_providers_vitisai.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake index 3d7f631e42f85..62eebb59bbd37 100644 --- a/cmake/onnxruntime_providers_vitisai.cmake +++ b/cmake/onnxruntime_providers_vitisai.cmake @@ -53,6 +53,10 @@ target_compile_options(onnxruntime_providers_vitisai PRIVATE -Wno-unused-parameter) endif(MSVC) + if(MSVC) + target_link_options(onnxruntime_providers_vitisai PRIVATE "/NODEFAULTLIB:libucrt.lib" "/DEFAULTLIB:ucrt.lib") + endif(MSVC) + set_target_properties(onnxruntime_providers_vitisai PROPERTIES FOLDER "ONNXRuntime") set_target_properties(onnxruntime_providers_vitisai PROPERTIES LINKER_LANGUAGE CXX) From 5ba2a1ed9798ea79c38162cd8d9b2e8eeba51de9 Mon Sep 17 00:00:00 2001 From: mingyue Date: Thu, 9 Jan 2025 22:32:13 -0700 Subject: [PATCH 06/10] fix rebase error --- onnxruntime/core/providers/vitisai/imp/global_api.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index b1eeb49dfd8dc..be068812af84b 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -49,6 +49,8 @@ int create_ep_context_nodes_c( vaip_core::DllSafe>* ret_value); std::vector>* compile_onnx_model_with_options_c( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); +std::vector>* compile_onnx_model_vitisai_ep_with_error_handling_c( + const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func); int vitisai_ep_on_run_start_c( const std::vector>& eps, const void* state, vaip_core::DllSafe (*get_config_entry)(const void* state, const char* entry_name)); @@ -56,6 +58,9 @@ int vitisai_ep_set_ep_dynamic_options_c( const std::vector>& eps, const char* const* keys, const char* const* values, size_t kv_len); +void profiler_collect_c( + std::vector& api_events, + std::vector& kernel_events); }; vaip_core::OrtApiForVaip* create_org_api_hook(); struct OrtVitisAIEpAPI { @@ -94,10 +99,12 @@ struct OrtVitisAIEpAPI { this->initialize_onnxruntime_vitisai_ep = initialize_onnxruntime_vitisai_ep_c; this->compile_onnx_model_with_options = compile_onnx_model_with_options_c; + this->compile_onnx_model_vitisai_ep_with_error_handling = compile_onnx_model_vitisai_ep_with_error_handling_c; this->create_ep_context_nodes = create_ep_context_nodes_c; this->vitisai_ep_on_run_start = vitisai_ep_on_run_start_c; this->vitisai_ep_set_ep_dynamic_options = vitisai_ep_set_ep_dynamic_options_c; this->vaip_get_version = vaip_get_version_c; + this->profiler_collect = profiler_collect_c; auto& env = Provider_GetHost()->Env__Default(); auto& logger = *Provider_GetHost()->LoggingManager_GetDefaultLogger(); From 064cc6c25d6d54151ac79dc34d761db3ce40925e Mon Sep 17 00:00:00 2001 From: mingyue Date: Mon, 13 Jan 2025 19:49:14 -0700 Subject: [PATCH 07/10] fix compile error --- onnxruntime/core/providers/vitisai/imp/global_api.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index be068812af84b..fd573b9340774 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -544,11 +544,6 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { the_global_api.graph_remove_initialized_tensor = [](Graph& graph, const std::string& tensor_name) { graph.RemoveInitializedTensor(tensor_name); }; - if (!s_library_vitisaiep.vaip_get_version) { - return reinterpret_cast(&(the_global_api.host_)); - } else { - return &the_global_api; - } the_global_api.vaip_xcompiler_compile = s_library_vitisaiep.vaip_xcompiler_compile; the_global_api.vaip_get_default_config = s_library_vitisaiep.vaip_get_default_config; the_global_api.vaip_get_pattern_as_binary = s_library_vitisaiep.vaip_get_pattern_as_binary; From afe6477856f9dccdf5d5279e5ff54be96d81e543 Mon Sep 17 00:00:00 2001 From: mingyue Date: Tue, 14 Jan 2025 03:57:11 -0700 Subject: [PATCH 08/10] use internal vaip url --- cmake/deps.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index e8e0cd03bf5dd..8fc348dd335d9 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,4 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b -vaip;https://gitenterprise.xilinx.com/VitisAI/vaip.git;win24_open_src +vaip;https://gitenterprise.xilinx.com/VitisAI/vaip.git;cp_dev_open_src From 41689249e3c169f59dff65d191071700b5972f66 Mon Sep 17 00:00:00 2001 From: mingyue Date: Wed, 29 Jan 2025 00:06:52 -0700 Subject: [PATCH 09/10] vaip url change to github --- cmake/deps.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index 8fc348dd335d9..d5ba9c48fe46e 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,4 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b -vaip;https://gitenterprise.xilinx.com/VitisAI/vaip.git;cp_dev_open_src +vaip;https://github.com/amd/vaip.git;main From 4d08552af6023ad661e533fc7e866e050bc8d242 Mon Sep 17 00:00:00 2001 From: mingyue Date: Fri, 7 Feb 2025 21:00:23 -0700 Subject: [PATCH 10/10] change vaip in cmake/deps.txt point to SHA1 --- cmake/deps.txt | 2 +- cmake/onnxruntime_providers_vitisai.cmake | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index d5ba9c48fe46e..482f2496cf3fb 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,4 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b -vaip;https://github.com/amd/vaip.git;main +vaip;https://github.com/amd/vaip/archive/refs/tags/v1.0.0-rc0.zip;e084b71c0277e2e3263179a4682300d427ff83e7 diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake index 62eebb59bbd37..a664ecb0e3b75 100644 --- a/cmake/onnxruntime_providers_vitisai.cmake +++ b/cmake/onnxruntime_providers_vitisai.cmake @@ -2,10 +2,8 @@ # Licensed under the MIT License. FetchContent_Declare( vaip - GIT_REPOSITORY ${DEP_URL_vaip} - GIT_TAG ${DEP_SHA1_vaip} - GIT_SUBMODULES_RECURSE FALSE - GIT_SHALLOW TRUE + URL ${DEP_URL_vaip} + URL_HASH SHA1=${DEP_SHA1_vaip} OVERRIDE_FIND_PACKAGE ) find_package(vaip)