Skip to content

[VitisAI] Add vaip Integration Using FetchContent (Cherry-pick of PR#22038 to win-ort-main branch) #23608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wonchung-microsoft merged 10 commits into from
Feb 10, 2025
Merged

[VitisAI] Add vaip Integration Using FetchContent (Cherry-pick of PR#22038 to win-ort-main branch) #23608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7cbcf5c
refactor VitisAI EP for opensource
wcy123 Aug 20, 2024
6eeafca
Add some VitisAI EP C APIs
Aug 26, 2024
def49fe
fix rebase error
Nov 7, 2024
20e4374
use internal vaip for rebase test
Nov 14, 2024
dcd9931
onnxruntime_providers_vitisai.dll enable shared link Hybrid CRT
Nov 15, 2024
5ba2a1e
fix rebase error
Jan 10, 2025
064cc6c
fix compile error
Jan 14, 2025
afe6477
use internal vaip url
Jan 14, 2025
4168924
vaip url change to github
Jan 29, 2025
4d08552
change vaip in cmake/deps.txt point to SHA1
Feb 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmake/deps.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v
cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1
dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43
kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b
vaip;https://github.com/amd/vaip/archive/refs/tags/v1.0.0-rc0.zip;e084b71c0277e2e3263179a4682300d427ff83e7
14 changes: 13 additions & 1 deletion cmake/onnxruntime_providers_vitisai.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
FetchContent_Declare(
vaip
URL ${DEP_URL_vaip}
URL_HASH SHA1=${DEP_SHA1_vaip}
OVERRIDE_FIND_PACKAGE
)
find_package(vaip)

if ("${GIT_COMMIT_ID}" STREQUAL "")
execute_process(
Expand All @@ -21,10 +28,11 @@
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs})
onnxruntime_add_shared_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs})
onnxruntime_add_include_to_target(onnxruntime_providers_vitisai ${ONNXRUNTIME_PROVIDERS_SHARED} ${GSL_TARGET} safeint_interface flatbuffers::flatbuffers)
target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED})
target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED} onnxruntime_vitisai_ep::onnxruntime_vitisai_ep)
if(MSVC)
onnxruntime_add_include_to_target(onnxruntime_providers_vitisai dbghelp)
set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/vitisai/symbols.def")
target_sources(onnxruntime_providers_vitisai PRIVATE ${vaip_BINARY_DIR}/onnxruntime_vitisai_ep/onnxruntime_vitisai_ep.def)
else(MSVC)
set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/vitisai/version_script.lds -Xlinker --gc-sections")
endif(MSVC)
Expand All @@ -43,6 +51,10 @@
target_compile_options(onnxruntime_providers_vitisai PRIVATE -Wno-unused-parameter)
endif(MSVC)

if(MSVC)
target_link_options(onnxruntime_providers_vitisai PRIVATE "/NODEFAULTLIB:libucrt.lib" "/DEFAULTLIB:ucrt.lib")

@snnn snnn Feb 7, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why this line is needed?

@wonchung-microsoft wonchung-microsoft Feb 10, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quote from AMD regarding this:

Linking Hybrid CRT is intended to optimize resource management and improve performance. Hybrid CRT (C Runtime) may combine different runtime libraries to optimize specific computational tasks, reduce unnecessary overhead, and potentially allow better memory management.
Among the models we support, some invoke custom ops implemented on the CPU. In our experiments, /MT + Hybrid CRT achieves the best performance.

I believe this is primarily for performance after AMD's internal experiments with vaip.

endif(MSVC)

set_target_properties(onnxruntime_providers_vitisai PROPERTIES FOLDER "ONNXRuntime")
set_target_properties(onnxruntime_providers_vitisai PROPERTIES LINKER_LANGUAGE CXX)

Expand Down
79 changes: 63 additions & 16 deletions onnxruntime/core/providers/vitisai/imp/global_api.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,27 @@ using namespace onnxruntime;
#define LIBRARY_PREFIX "lib"
#define LIBRARY_EXTENSION ".so"
#endif

extern "C" {
void initialize_onnxruntime_vitisai_ep_c(vaip_core::OrtApiForVaip* api, std::vector<OrtCustomOpDomain*>& ret_domain);
uint32_t vaip_get_version_c();
int create_ep_context_nodes_c(
const std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>& eps,
vaip_core::DllSafe<std::vector<Node*>>* ret_value);
std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>* compile_onnx_model_with_options_c(
const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options);
std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>* compile_onnx_model_vitisai_ep_with_error_handling_c(
const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func);
int vitisai_ep_on_run_start_c(
const std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>& eps, const void* state,
vaip_core::DllSafe<std::string> (*get_config_entry)(const void* state, const char* entry_name));
int vitisai_ep_set_ep_dynamic_options_c(
const std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>& eps,
const char* const* keys,
const char* const* values, size_t kv_len);
void profiler_collect_c(
std::vector<EventInfo>& api_events,
std::vector<EventInfo>& kernel_events);
};
vaip_core::OrtApiForVaip* create_org_api_hook();
struct OrtVitisAIEpAPI {
void (*initialize_onnxruntime_vitisai_ep)(vaip_core::OrtApiForVaip* api, std::vector<OrtCustomOpDomain*>& ret_domain);
Expand All @@ -50,7 +70,7 @@ struct OrtVitisAIEpAPI {
std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>* (*compile_onnx_model_vitisai_ep_with_error_handling)(
const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func);
uint32_t (*vaip_get_version)();
void (*create_ep_context_nodes)(
int (*create_ep_context_nodes)(
const std::vector<std::unique_ptr<vaip_core::ExecutionProvider>>& eps,
vaip_core::DllSafe<std::vector<Node*>>* ret_value) = nullptr;
int (*vitisai_ep_on_run_start)(
Expand All @@ -63,34 +83,54 @@ struct OrtVitisAIEpAPI {
void (*profiler_collect)(
std::vector<EventInfo>& api_events,
std::vector<EventInfo>& kernel_events);
int (*vaip_xcompiler_compile)(const char* input_xmodel,
size_t input_xmodel_size,
const char* config_xmodel,
size_t config_xmodel_size, void* state,
void (*k)(void*, void*, size_t)) = nullptr;
const char* (*vaip_get_default_config)() = nullptr;
int (*vaip_get_pattern_as_binary)(const char* name, void* state, void (*k)(void*, void*, size_t)) = nullptr;
void (*vaip_get_pattern_list)(void* state, void (*k)(void*, void*, size_t)) = nullptr;
int (*vaip_get_mem_xclbin)(const char* name, void* state, void (*k)(void*, void*, size_t)) = nullptr;
bool (*vaip_has_mem_xclbin)(const char* name) = nullptr;
void Ensure() {
if (handle_)
return;

this->initialize_onnxruntime_vitisai_ep = initialize_onnxruntime_vitisai_ep_c;
this->compile_onnx_model_with_options = compile_onnx_model_with_options_c;
this->compile_onnx_model_vitisai_ep_with_error_handling = compile_onnx_model_vitisai_ep_with_error_handling_c;
this->create_ep_context_nodes = create_ep_context_nodes_c;
this->vitisai_ep_on_run_start = vitisai_ep_on_run_start_c;
this->vitisai_ep_set_ep_dynamic_options = vitisai_ep_set_ep_dynamic_options_c;
this->vaip_get_version = vaip_get_version_c;
this->profiler_collect = profiler_collect_c;

auto& env = Provider_GetHost()->Env__Default();
auto& logger = *Provider_GetHost()->LoggingManager_GetDefaultLogger();

#ifdef _WIN32
// this dll is already linked to the executable, normally a test program
handle_ = reinterpret_cast<void*>(GetModuleHandle(TEXT("onnxruntime_vitisai_ep.dll")));
auto status = Status::OK();
if (!handle_) {
auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION);
ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_));
status = env.LoadDynamicLibrary(full_path, true, &handle_);
if (!status.IsOK()) {
LOGS(logger, VERBOSE) << "cannot load onnxruntime_vitisai_ep.dll, can only deploy ep_context onnx model.";
return;
}
}
#else
auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION);
ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_));
#endif
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "initialize_onnxruntime_vitisai_ep", (void**)&initialize_onnxruntime_vitisai_ep));
auto status1 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_error_handling", (void**)&compile_onnx_model_vitisai_ep_with_error_handling);
auto status2 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_options", (void**)&compile_onnx_model_with_options);
if ((!status1.IsOK()) && (!status2.IsOK())) {
::onnxruntime::LogRuntimeError(0, status2, __FILE__, static_cast<const char*>(__FUNCTION__), __LINE__);
ORT_THROW(status2);
}
std::ignore = env.GetSymbolFromLibrary(handle_, "vaip_get_version",
(void**)&vaip_get_version);
std::ignore = env.GetSymbolFromLibrary(handle_, "profiler_collect", (void**)&profiler_collect);
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "create_ep_context_nodes", (void**)&create_ep_context_nodes));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_on_run_start", (void**)&vitisai_ep_on_run_start));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_set_ep_dynamic_options", (void**)&vitisai_ep_set_ep_dynamic_options));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_xcompiler_compile", (void**)&vaip_xcompiler_compile));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_default_config", (void**)&vaip_get_default_config));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_pattern_as_binary", (void**)&vaip_get_pattern_as_binary));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_pattern_list", (void**)&vaip_get_pattern_list));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_get_mem_xclbin", (void**)&vaip_get_mem_xclbin));
ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_has_mem_xclbin", (void**)&vaip_has_mem_xclbin));
}
void Clear() {
if (handle_) {
Expand Down Expand Up @@ -484,6 +524,7 @@ vaip_core::OrtApiForVaip* create_org_api_hook() {
}
};
the_global_api.node_arg_external_location = vaip::node_arg_external_location;

the_global_api.model_to_proto = [](onnxruntime::Model& model) { return model.ToProto().release(); };
the_global_api.model_proto_serialize_as_string = [](ONNX_NAMESPACE::ModelProto& model_proto) {
return vaip_core::DllSafe(model_proto.SerializeAsString());
Expand All @@ -503,6 +544,12 @@ vaip_core::OrtApiForVaip* create_org_api_hook() {
the_global_api.graph_remove_initialized_tensor = [](Graph& graph, const std::string& tensor_name) {
graph.RemoveInitializedTensor(tensor_name);
};
the_global_api.vaip_xcompiler_compile = s_library_vitisaiep.vaip_xcompiler_compile;
the_global_api.vaip_get_default_config = s_library_vitisaiep.vaip_get_default_config;
the_global_api.vaip_get_pattern_as_binary = s_library_vitisaiep.vaip_get_pattern_as_binary;
the_global_api.vaip_get_pattern_list = s_library_vitisaiep.vaip_get_pattern_list;
the_global_api.vaip_get_mem_xclbin = s_library_vitisaiep.vaip_get_mem_xclbin;
the_global_api.vaip_has_mem_xclbin = s_library_vitisaiep.vaip_has_mem_xclbin;
if (!s_library_vitisaiep.vaip_get_version) {
return reinterpret_cast<vaip_core::OrtApiForVaip*>(&(the_global_api.host_));
} else {
Expand Down
12 changes: 11 additions & 1 deletion onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,14 +235,24 @@ struct OrtApiForVaip {
DllSafe<std::string> (*model_proto_serialize_as_string)(ModelProto& model_proto); // [96]
void (*model_proto_delete)(ModelProto* p); // [97]
DllSafe<std::string> (*attr_proto_release_string)(AttributeProto* attr); // [98]
bool (*is_profiling_enabled)(void* session_options); // [99] // [98]
bool (*is_profiling_enabled)(void* session_options); // [99]
TensorProto* (*tensor_proto_new_i4)(const std::string& name,
const std::vector<int64_t>& shape,
const std::vector<int8_t>& data); // [100]
TensorProto* (*tensor_proto_new_u4)(const std::string& name,
const std::vector<int64_t>& shape,
const std::vector<uint8_t>& data); // [101]
void (*graph_remove_initialized_tensor)(Graph& graph, const std::string& tensor_name); // [102]
int (*vaip_xcompiler_compile)(const char* input_xmodel,
size_t input_xmodel_size,
const char* config_xmodel,
size_t config_xmodel_size, void* state,
void (*k)(void*, void*, size_t)); // [103]
const char* (*vaip_get_default_config)(); // [104]
int (*vaip_get_pattern_as_binary)(const char* name, void* state, void (*k)(void*, void*, size_t)); // [105]
void (*vaip_get_pattern_list)(void* state, void (*k)(void*, void*, size_t)); // [106]
int (*vaip_get_mem_xclbin)(const char* name, void* state, void (*k)(void*, void*, size_t)); // [107]
bool (*vaip_has_mem_xclbin)(const char* name); // [108]
};

#ifndef USE_VITISAI
Expand Down