From cc8b267ce2270659d126c7f416a99a08cb9e4b53 Mon Sep 17 00:00:00 2001 From: Jaswanth51 Date: Tue, 2 Sep 2025 11:32:23 +0530 Subject: [PATCH] Revert "Sync with Microsoft ONNX Runtime - 01/09/2025 (#801)" This reverts commit 2f1ad9d04d6c900d3c2749838f8196e720456e81. --- cmake/CMakeLists.txt | 8 +- .../external/onnxruntime_external_deps.cmake | 61 ++- cmake/onnxruntime.cmake | 13 +- cmake/onnxruntime_common.cmake | 57 ++- cmake/onnxruntime_java.cmake | 4 +- cmake/onnxruntime_nodejs.cmake | 1 - cmake/onnxruntime_unittests.cmake | 4 - .../cpuinfo/patch_vcpkg_arm64ec_support.patch | 91 ---- .../cpuinfo/patch_vcpkg_arm64ec_support.patch | 91 ---- cmake/vcpkg-ports/cpuinfo/portfile.cmake | 1 - .../NativeMethods.shared.cs | 98 ---- .../Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs | 40 -- .../EpCompatibilityTests.cs | 49 -- .../providers/cann/cann_provider_options.h | 2 - .../core/providers/utils/ort_graph_to_proto.h | 8 +- .../core/session/onnxruntime_c_api.h | 3 +- .../core/session/onnxruntime_cxx_api.h | 125 +---- .../core/session/onnxruntime_cxx_inline.h | 20 - .../onnxruntime_ep_device_ep_metadata_keys.h | 5 +- .../main/java/ai/onnxruntime/OnnxRuntime.java | 18 +- .../java/ai/onnxruntime/OrtEnvironment.java | 82 +-- .../main/java/ai/onnxruntime/OrtEpDevice.java | 117 ----- .../ai/onnxruntime/OrtHardwareDevice.java | 156 ------ .../OrtModelCompilationOptions.java | 280 ----------- .../main/java/ai/onnxruntime/OrtSession.java | 78 +-- .../src/main/java/ai/onnxruntime/OrtUtil.java | 51 +- .../ai/onnxruntime/providers/CoreMLFlags.java | 4 +- .../ai/onnxruntime/providers/NNAPIFlags.java | 4 +- .../onnxruntime/{ => providers}/OrtFlags.java | 4 +- java/src/main/native/OrtJniUtil.c | 30 -- java/src/main/native/OrtJniUtil.h | 2 - .../main/native/ai_onnxruntime_OnnxRuntime.c | 13 - .../native/ai_onnxruntime_OrtEnvironment.c | 70 --- .../main/native/ai_onnxruntime_OrtEpDevice.c | 82 --- .../native/ai_onnxruntime_OrtHardwareDevice.c | 96 ---- ...i_onnxruntime_OrtModelCompilationOptions.c | 193 -------- ...ai_onnxruntime_OrtSession_SessionOptions.c | 53 +- .../java/ai/onnxruntime/CompileApiTest.java | 53 -- .../java/ai/onnxruntime/EpDeviceTest.java | 123 ----- js/node/src/inference_session_wrap.cc | 2 +- .../contrib_ops/cpu/bert/gqa_attention_base.h | 15 +- .../cpu/moe/moe_quantization_cpu.cc | 11 +- .../quantization/dynamic_quantize_matmul.cc | 17 +- .../core/common/cpuid_arch_definition.h | 2 +- onnxruntime/core/graph/abi_graph_types.h | 10 + onnxruntime/core/graph/ep_api_types.cc | 29 +- onnxruntime/core/graph/ep_api_types.h | 3 + .../core/graph/model_editor_api_types.h | 5 + .../core/mlas/lib/kleidiai/sgemm_kleidiai.cpp | 73 ++- .../providers/cann/cann_execution_provider.cc | 9 +- .../cann/cann_execution_provider_info.cc | 4 - .../cann/cann_execution_provider_info.h | 1 - .../providers/cann/cann_provider_factory.cc | 2 - .../nv_tensorrt_rtx/nv_execution_provider.cc | 151 +++++- .../nv_tensorrt_rtx/nv_execution_provider.h | 2 + .../qnn/builder/opbuilder/conv_op_builder.cc | 14 +- .../shared_library/provider_ort_api_init.cc | 4 +- .../core/providers/vitisai/imp/global_api.cc | 6 +- onnxruntime/core/session/onnxruntime_c_api.cc | 38 +- onnxruntime/core/session/ort_apis.h | 2 +- .../plugin_ep/ep_factory_provider_bridge.cc | 7 - .../plugin_ep/ep_factory_provider_bridge.h | 15 +- .../core/session/plugin_ep/ep_library.h | 1 - .../plugin_ep/ep_library_provider_bridge.cc | 4 +- .../plugin_ep/ep_library_provider_bridge.h | 9 +- .../core/session/provider_bridge_ort.cc | 1 - onnxruntime/core/session/utils.cc | 5 +- .../python/onnxruntime_pybind_state.cc | 19 +- .../execution_providers/qnn/preprocess.py | 24 +- onnxruntime/test/autoep/library/ep_arena.h | 3 - onnxruntime/test/contrib_ops/moe_test.cc | 55 -- .../test/framework/ep_compatibility_test.cc | 29 -- .../test/platform/device_discovery_test.cc | 4 +- ...nnxruntime_test_python_ep_compatibility.py | 46 -- ...me_test_python_nv_tensorrt_rtx_ep_tests.py | 468 ------------------ .../custom_op_library/custom_op_library.cc | 2 +- .../github/windows/extract_nuget_files.ps1 | 148 +++--- .../windows/extract_nuget_files_gpu.ps1 | 86 +--- 78 files changed, 509 insertions(+), 3007 deletions(-) delete mode 100644 cmake/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch delete mode 100644 cmake/vcpkg-ports/cpuinfo/patch_vcpkg_arm64ec_support.patch delete mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/EpCompatibilityTests.cs delete mode 100644 java/src/main/java/ai/onnxruntime/OrtEpDevice.java delete mode 100644 java/src/main/java/ai/onnxruntime/OrtHardwareDevice.java delete mode 100644 java/src/main/java/ai/onnxruntime/OrtModelCompilationOptions.java rename java/src/main/java/ai/onnxruntime/{ => providers}/OrtFlags.java (88%) delete mode 100644 java/src/main/native/ai_onnxruntime_OrtEpDevice.c delete mode 100644 java/src/main/native/ai_onnxruntime_OrtHardwareDevice.c delete mode 100644 java/src/main/native/ai_onnxruntime_OrtModelCompilationOptions.c delete mode 100644 java/src/test/java/ai/onnxruntime/CompileApiTest.java delete mode 100644 java/src/test/java/ai/onnxruntime/EpDeviceTest.java delete mode 100644 onnxruntime/test/python/onnxruntime_test_python_ep_compatibility.py delete mode 100644 onnxruntime/test/python/onnxruntime_test_python_nv_tensorrt_rtx_ep_tests.py diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 40e6a8da28e45..98548957d0b42 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1607,6 +1607,7 @@ if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") endif() endif() + #Now the 'onnxruntime_EXTERNAL_LIBRARIES' variable should be sealed. It will be used in onnxruntime.cmake which will be included in the next. #The order of the following targets matters. Right depends on left. If target A appears before target B. Then A.cmake can not use variables defined in B.cmake. set(ONNXRUNTIME_CMAKE_FILES onnxruntime_flatbuffers onnxruntime_common onnxruntime_mlas onnxruntime_graph onnxruntime_lora onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session ${ONNXRUNTIME_EAGER_CMAKE_FILE_NAME}) @@ -1622,6 +1623,9 @@ if (onnxruntime_USE_WINML) list(APPEND ONNXRUNTIME_CMAKE_FILES winml) endif() # if (onnxruntime_USE_WINML) +if (onnxruntime_BUILD_APPLE_FRAMEWORK AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS|visionOS|tvOS") + message(FATAL_ERROR "onnxruntime_BUILD_APPLE_FRAMEWORK can only be enabled for macOS or iOS or visionOS or tvOS.") +endif() list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime) if (onnxruntime_BUILD_JAVA) @@ -1686,8 +1690,8 @@ if (WIN32 AND NOT GDK_PLATFORM AND NOT CMAKE_CROSSCOMPILING) endif() endif() -foreach(onnxruntime_cmake_file ${ONNXRUNTIME_CMAKE_FILES}) - include(${onnxruntime_cmake_file}.cmake) +foreach(target_name ${ONNXRUNTIME_CMAKE_FILES}) + include(${target_name}.cmake) endforeach() if (UNIX) option(BUILD_PKGCONFIG_FILES "Build and install pkg-config files" ON) diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 827be3e6dea2a..3095968795d1a 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -313,32 +313,41 @@ onnxruntime_fetchcontent_makeavailable(nlohmann_json) if (onnxruntime_ENABLE_CPUINFO) # Adding pytorch CPU info library # TODO!! need a better way to find out the supported architectures - set(CPUINFO_SUPPORTED FALSE) + list(LENGTH CMAKE_OSX_ARCHITECTURES CMAKE_OSX_ARCHITECTURES_LEN) if (APPLE) - list(LENGTH CMAKE_OSX_ARCHITECTURES CMAKE_OSX_ARCHITECTURES_LEN) if (CMAKE_OSX_ARCHITECTURES_LEN LESS_EQUAL 1) set(CPUINFO_SUPPORTED TRUE) - else() - message(WARNING "cpuinfo is not supported when CMAKE_OSX_ARCHITECTURES has more than one value.") + elseif (onnxruntime_BUILD_APPLE_FRAMEWORK) + # We stitch multiple static libraries together when onnxruntime_BUILD_APPLE_FRAMEWORK is true, + # but that would not work for universal static libraries + message(FATAL_ERROR "universal binary is not supported for apple framework") endif() - elseif (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - # if xnnpack is enabled in a wasm build it needs clog from cpuinfo, but we won't internally use cpuinfo. - if (onnxruntime_USE_XNNPACK) - set(CPUINFO_SUPPORTED TRUE) - endif() - elseif (WIN32) - set(CPUINFO_SUPPORTED TRUE) else() - if (onnxruntime_target_platform MATCHES "^(i[3-6]86|AMD64|x86(_64)?|armv[5-8].*|aarch64|arm64)$") - set(CPUINFO_SUPPORTED TRUE) + # if xnnpack is enabled in a wasm build it needs clog from cpuinfo, but we won't internally use cpuinfo + # so we don't set CPUINFO_SUPPORTED in the CXX flags below. + if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND NOT onnxruntime_USE_XNNPACK) + set(CPUINFO_SUPPORTED FALSE) else() - message(WARNING "Target processor architecture \"${onnxruntime_target_platform}\" is not supported in cpuinfo.") + set(CPUINFO_SUPPORTED TRUE) + endif() + if (WIN32) + # There's an error when linking with cpuinfo on arm64ec with a vcpkg build (--use_vcpkg). + # TODO Fix it and then re-enable cpuinfo on arm64ec. + if (onnxruntime_target_platform STREQUAL "ARM64EC") + set(CPUINFO_SUPPORTED FALSE) + else() + set(CPUINFO_SUPPORTED TRUE) + endif() + elseif (NOT ${onnxruntime_target_platform} MATCHES "^(i[3-6]86|AMD64|x86(_64)?|armv[5-8].*|aarch64|arm64)$") + message(WARNING + "Target processor architecture \"${onnxruntime_target_platform}\" is not supported in cpuinfo. " + "cpuinfo not included." + ) + set(CPUINFO_SUPPORTED FALSE) endif() endif() - - if(NOT CPUINFO_SUPPORTED) - message(WARNING "onnxruntime_ENABLE_CPUINFO was set but cpuinfo is not supported.") - endif() +else() + set(CPUINFO_SUPPORTED FALSE) endif() if (CPUINFO_SUPPORTED) @@ -349,26 +358,23 @@ if (CPUINFO_SUPPORTED) # if this is a wasm build with xnnpack (only type of wasm build where cpuinfo is involved) # we do not use cpuinfo in ORT code, so don't define CPUINFO_SUPPORTED. - if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND onnxruntime_USE_XNNPACK) - else() - add_compile_definitions(CPUINFO_SUPPORTED) + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + string(APPEND CMAKE_CXX_FLAGS " -DCPUINFO_SUPPORTED") endif() + set(CPUINFO_BUILD_TOOLS OFF CACHE INTERNAL "") set(CPUINFO_BUILD_UNIT_TESTS OFF CACHE INTERNAL "") set(CPUINFO_BUILD_MOCK_TESTS OFF CACHE INTERNAL "") set(CPUINFO_BUILD_BENCHMARKS OFF CACHE INTERNAL "") if (onnxruntime_target_platform STREQUAL "ARM64EC" OR onnxruntime_target_platform STREQUAL "ARM64") - message(STATUS "Applying patches for Windows ARM64/ARM64EC in cpuinfo") + message(STATUS "Applying a patch for Windows ARM64/ARM64EC in cpuinfo") onnxruntime_fetchcontent_declare( pytorch_cpuinfo URL ${DEP_URL_pytorch_cpuinfo} URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo} EXCLUDE_FROM_ALL - PATCH_COMMAND - ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_cpuinfo_h_for_arm64ec.patch && - # https://github.com/pytorch/cpuinfo/pull/324 - ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch + PATCH_COMMAND ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_cpuinfo_h_for_arm64ec.patch FIND_PACKAGE_ARGS NAMES cpuinfo ) else() @@ -578,7 +584,8 @@ endif() set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} ${WIL_TARGET} nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface - flatbuffers::flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS} date::date Eigen3::Eigen) + flatbuffers::flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS} date::date + ${ONNXRUNTIME_CLOG_TARGET_NAME} Eigen3::Eigen) # The source code of onnx_proto is generated, we must build this lib first before starting to compile the other source code that uses ONNX protobuf types. # The other libs do not have the problem. All the sources are already there. We can compile them in any order. diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index e1d98109208d4..010696a61022c 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -350,19 +350,8 @@ if (winml_is_inbox) endif() endif() -# Assemble the Apple static framework +# Assemble the Apple static framework (iOS and macOS) if(onnxruntime_BUILD_APPLE_FRAMEWORK) - if (NOT CMAKE_SYSTEM_NAME MATCHES "Darwin|iOS|visionOS|tvOS") - message(FATAL_ERROR "onnxruntime_BUILD_APPLE_FRAMEWORK can only be enabled for macOS or iOS or visionOS or tvOS.") - endif() - - list(LENGTH CMAKE_OSX_ARCHITECTURES CMAKE_OSX_ARCHITECTURES_LEN) - if (CMAKE_OSX_ARCHITECTURES_LEN GREATER 1) - # We stitch multiple static libraries together when onnxruntime_BUILD_APPLE_FRAMEWORK is true, - # but that would not work for universal static libraries - message(FATAL_ERROR "universal binary is not supported for apple framework") - endif() - # when building for mac catalyst, the CMAKE_OSX_SYSROOT is set to MacOSX as well, to avoid duplication, # we specify as `-macabi` in the name of the output static apple framework directory. if (PLATFORM_NAME STREQUAL "macabi") diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 0218994e537a0..d927489372e7c 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -194,10 +194,59 @@ if(APPLE) target_link_libraries(onnxruntime_common PRIVATE "-framework Foundation") endif() -if(CPUINFO_SUPPORTED) - # Link cpuinfo if supported - onnxruntime_add_include_to_target(onnxruntime_common cpuinfo::cpuinfo) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES cpuinfo::cpuinfo) +if(MSVC) + if(onnxruntime_target_platform STREQUAL "ARM64") + set(ARM64 TRUE) + elseif (onnxruntime_target_platform STREQUAL "ARM") + set(ARM TRUE) + elseif(onnxruntime_target_platform STREQUAL "x64") + set(X64 TRUE) + elseif(onnxruntime_target_platform STREQUAL "x86") + set(X86 TRUE) + endif() +elseif(APPLE) + if(CMAKE_OSX_ARCHITECTURES_LEN LESS_EQUAL 1) + set(X64 TRUE) + endif() +elseif(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + if (CMAKE_SYSTEM_NAME STREQUAL "Android") + if (CMAKE_ANDROID_ARCH_ABI STREQUAL "armeabi-v7a") + set(ARM TRUE) + elseif (CMAKE_ANDROID_ARCH_ABI STREQUAL "arm64-v8a") + set(ARM64 TRUE) + elseif (CMAKE_ANDROID_ARCH_ABI STREQUAL "x86_64") + set(X86_64 TRUE) + elseif (CMAKE_ANDROID_ARCH_ABI STREQUAL "x86") + set(X86 TRUE) + endif() + else() + execute_process( + COMMAND ${CMAKE_C_COMPILER} -dumpmachine + OUTPUT_VARIABLE dumpmachine_output + ERROR_QUIET + ) + if(dumpmachine_output MATCHES "^arm64.*") + set(ARM64 TRUE) + elseif(dumpmachine_output MATCHES "^arm.*") + set(ARM TRUE) + elseif(dumpmachine_output MATCHES "^aarch64.*") + set(ARM64 TRUE) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^riscv64.*") + set(RISCV64 TRUE) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86?)$") + set(X86 TRUE) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64)$") + set(X86_64 TRUE) + endif() + endif() +endif() + +if (RISCV64 OR ARM64 OR ARM OR X86 OR X64 OR X86_64) + # Link cpuinfo if supported + if (CPUINFO_SUPPORTED) + onnxruntime_add_include_to_target(onnxruntime_common cpuinfo::cpuinfo) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES cpuinfo::cpuinfo ${ONNXRUNTIME_CLOG_TARGET_NAME}) + endif() endif() if (NOT onnxruntime_BUILD_SHARED_LIB) diff --git a/cmake/onnxruntime_java.cmake b/cmake/onnxruntime_java.cmake index 7da63b523be70..6b638b3e5d8bc 100644 --- a/cmake/onnxruntime_java.cmake +++ b/cmake/onnxruntime_java.cmake @@ -159,7 +159,7 @@ if (WIN32) if(NOT onnxruntime_ENABLE_STATIC_ANALYSIS) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_PACKAGE_LIB_DIR}/$) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_PACKAGE_JNI_DIR}/$) - if (TARGET onnxruntime_providers_shared) + if (onnxruntime_USE_CUDA OR onnxruntime_USE_DNNL OR onnxruntime_USE_OPENVINO OR onnxruntime_USE_TENSORRT OR (onnxruntime_USE_QNN AND NOT onnxruntime_BUILD_QNN_EP_STATIC_LIB)) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_PACKAGE_LIB_DIR}/$) endif() if (onnxruntime_USE_CUDA) @@ -207,7 +207,7 @@ if (WIN32) else() add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_PACKAGE_LIB_DIR}/$) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_PACKAGE_JNI_DIR}/$) - if (TARGET onnxruntime_providers_shared) + if (onnxruntime_USE_CUDA OR onnxruntime_USE_DNNL OR onnxruntime_USE_OPENVINO OR onnxruntime_USE_TENSORRT OR (onnxruntime_USE_QNN AND NOT onnxruntime_BUILD_QNN_EP_STATIC_LIB)) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_PACKAGE_LIB_DIR}/$) endif() if (onnxruntime_USE_CUDA) diff --git a/cmake/onnxruntime_nodejs.cmake b/cmake/onnxruntime_nodejs.cmake index cce0810c5bbe8..b28bda6c94276 100644 --- a/cmake/onnxruntime_nodejs.cmake +++ b/cmake/onnxruntime_nodejs.cmake @@ -10,7 +10,6 @@ include(node_helper.cmake) # setup ARCH if (APPLE) - list(LENGTH CMAKE_OSX_ARCHITECTURES CMAKE_OSX_ARCHITECTURES_LEN) if (CMAKE_OSX_ARCHITECTURES_LEN GREATER 1) message(FATAL_ERROR "CMake.js does not support multi-architecture for macOS") endif() diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index b31849440c426..6847db64004ca 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1640,10 +1640,6 @@ if (NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") add_custom_command(TARGET onnxruntime_providers_qnn POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${QNN_LIB_FILES} ${JAVA_NATIVE_TEST_DIR}) endif() - if (WIN32) - set(EXAMPLE_PLUGIN_EP_DST_FILE_NAME $,$,$>) - add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ ${JAVA_NATIVE_TEST_DIR}/${EXAMPLE_PLUGIN_EP_DST_FILE_NAME}) - endif() # delegate to gradle's test runner diff --git a/cmake/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch b/cmake/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch deleted file mode 100644 index af0f039b6c2a3..0000000000000 --- a/cmake/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch +++ /dev/null @@ -1,91 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index aedc983..dab589e 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -72,6 +72,17 @@ IF(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND CPUINFO_TARGET_PROCESSOR STREQUAL "am - ENDIF() - IF(IS_APPLE_OS AND CMAKE_OSX_ARCHITECTURES MATCHES "^(x86_64|arm64.*)$") - SET(CPUINFO_TARGET_PROCESSOR "${CMAKE_OSX_ARCHITECTURES}") -+ELSEIF(MSVC AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.10") -+ # Use CMAKE_C_COMPILER_ARCHITECTURE_ID. MSVC values are documented as available since CMake 3.10. -+ IF(CMAKE_C_COMPILER_ARCHITECTURE_ID STREQUAL "X86") -+ SET(CPUINFO_TARGET_PROCESSOR "x86") -+ ELSEIF(CMAKE_C_COMPILER_ARCHITECTURE_ID STREQUAL "x64") -+ SET(CPUINFO_TARGET_PROCESSOR "x86_64") -+ ELSEIF(CMAKE_C_COMPILER_ARCHITECTURE_ID MATCHES "^(ARM64|ARM64EC)$") -+ SET(CPUINFO_TARGET_PROCESSOR "arm64") -+ ELSE() -+ MESSAGE(FATAL_ERROR "Unsupported MSVC compiler architecture ID \"${CMAKE_C_COMPILER_ARCHITECTURE_ID}\"") -+ ENDIF() - ELSEIF(CMAKE_GENERATOR MATCHES "^Visual Studio " AND CMAKE_VS_PLATFORM_NAME) - IF(CMAKE_VS_PLATFORM_NAME STREQUAL "Win32") - SET(CPUINFO_TARGET_PROCESSOR "x86") -@@ -88,7 +99,7 @@ ENDIF() - - # ---[ Build flags - SET(CPUINFO_SUPPORTED_PLATFORM TRUE) --IF(NOT CMAKE_SYSTEM_PROCESSOR) -+IF(NOT CPUINFO_TARGET_PROCESSOR) - IF(NOT IOS) - MESSAGE(WARNING - "Target processor architecture is not specified. " -@@ -201,12 +212,12 @@ IF(CPUINFO_SUPPORTED_PLATFORM) - src/arm/linux/chipset.c - src/arm/linux/midr.c - src/arm/linux/hwcap.c) -- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv[5-8]") -+ IF(CPUINFO_TARGET_PROCESSOR MATCHES "^armv[5-8]") - LIST(APPEND CPUINFO_SRCS src/arm/linux/aarch32-isa.c) - IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND ANDROID_ABI STREQUAL "armeabi") - SET_SOURCE_FILES_PROPERTIES(src/arm/linux/aarch32-isa.c PROPERTIES COMPILE_FLAGS -marm) - ENDIF() -- ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") -+ ELSEIF(CPUINFO_TARGET_PROCESSOR MATCHES "^(aarch64|arm64)$") - LIST(APPEND CPUINFO_SRCS src/arm/linux/aarch64-isa.c) - ENDIF() - ELSEIF(IS_APPLE_OS AND CPUINFO_TARGET_PROCESSOR MATCHES "arm64.*") -@@ -395,7 +406,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) - TARGET_COMPILE_DEFINITIONS(cpuinfo_mock PRIVATE _GNU_SOURCE=1) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv5te|armv7-a)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv5te|armv7-a)$") - ADD_EXECUTABLE(atm7029b-tablet-test test/mock/atm7029b-tablet.cc) - TARGET_INCLUDE_DIRECTORIES(atm7029b-tablet-test BEFORE PRIVATE test/mock) - TARGET_LINK_LIBRARIES(atm7029b-tablet-test PRIVATE cpuinfo_mock gtest) -@@ -577,7 +588,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) - ADD_TEST(NAME xperia-sl-test COMMAND xperia-sl-test) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv5te|armv7-a|aarch64)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv5te|armv7-a|aarch64)$") - ADD_EXECUTABLE(alcatel-revvl-test test/mock/alcatel-revvl.cc) - TARGET_INCLUDE_DIRECTORIES(alcatel-revvl-test BEFORE PRIVATE test/mock) - TARGET_LINK_LIBRARIES(alcatel-revvl-test PRIVATE cpuinfo_mock gtest) -@@ -774,7 +785,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) - ADD_TEST(NAME xperia-c4-dual-test COMMAND xperia-c4-dual-test) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(i686|x86_64)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(i686|x86_64)$") - ADD_EXECUTABLE(alldocube-iwork8-test test/mock/alldocube-iwork8.cc) - TARGET_INCLUDE_DIRECTORIES(alldocube-iwork8-test BEFORE PRIVATE test/mock) - TARGET_LINK_LIBRARIES(alldocube-iwork8-test PRIVATE cpuinfo_mock gtest) -@@ -831,7 +842,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_UNIT_TESTS) - ADD_TEST(NAME brand-string-test COMMAND brand-string-test) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") - ADD_LIBRARY(android_properties_interface STATIC test/name/android-properties-interface.c) - CPUINFO_TARGET_ENABLE_C99(android_properties_interface) - CPUINFO_TARGET_RUNTIME_LIBRARY(android_properties_interface) -@@ -879,7 +890,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_TOOLS) - TARGET_LINK_LIBRARIES(cache-info PRIVATE cpuinfo) - INSTALL(TARGETS cache-info RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - -- IF(CMAKE_SYSTEM_NAME MATCHES "^(Android|Linux)$" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") -+ IF(CMAKE_SYSTEM_NAME MATCHES "^(Android|Linux)$" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") - ADD_EXECUTABLE(auxv-dump tools/auxv-dump.c) - CPUINFO_TARGET_ENABLE_C99(auxv-dump) - CPUINFO_TARGET_RUNTIME_LIBRARY(auxv-dump) diff --git a/cmake/vcpkg-ports/cpuinfo/patch_vcpkg_arm64ec_support.patch b/cmake/vcpkg-ports/cpuinfo/patch_vcpkg_arm64ec_support.patch deleted file mode 100644 index af0f039b6c2a3..0000000000000 --- a/cmake/vcpkg-ports/cpuinfo/patch_vcpkg_arm64ec_support.patch +++ /dev/null @@ -1,91 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index aedc983..dab589e 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -72,6 +72,17 @@ IF(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND CPUINFO_TARGET_PROCESSOR STREQUAL "am - ENDIF() - IF(IS_APPLE_OS AND CMAKE_OSX_ARCHITECTURES MATCHES "^(x86_64|arm64.*)$") - SET(CPUINFO_TARGET_PROCESSOR "${CMAKE_OSX_ARCHITECTURES}") -+ELSEIF(MSVC AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.10") -+ # Use CMAKE_C_COMPILER_ARCHITECTURE_ID. MSVC values are documented as available since CMake 3.10. -+ IF(CMAKE_C_COMPILER_ARCHITECTURE_ID STREQUAL "X86") -+ SET(CPUINFO_TARGET_PROCESSOR "x86") -+ ELSEIF(CMAKE_C_COMPILER_ARCHITECTURE_ID STREQUAL "x64") -+ SET(CPUINFO_TARGET_PROCESSOR "x86_64") -+ ELSEIF(CMAKE_C_COMPILER_ARCHITECTURE_ID MATCHES "^(ARM64|ARM64EC)$") -+ SET(CPUINFO_TARGET_PROCESSOR "arm64") -+ ELSE() -+ MESSAGE(FATAL_ERROR "Unsupported MSVC compiler architecture ID \"${CMAKE_C_COMPILER_ARCHITECTURE_ID}\"") -+ ENDIF() - ELSEIF(CMAKE_GENERATOR MATCHES "^Visual Studio " AND CMAKE_VS_PLATFORM_NAME) - IF(CMAKE_VS_PLATFORM_NAME STREQUAL "Win32") - SET(CPUINFO_TARGET_PROCESSOR "x86") -@@ -88,7 +99,7 @@ ENDIF() - - # ---[ Build flags - SET(CPUINFO_SUPPORTED_PLATFORM TRUE) --IF(NOT CMAKE_SYSTEM_PROCESSOR) -+IF(NOT CPUINFO_TARGET_PROCESSOR) - IF(NOT IOS) - MESSAGE(WARNING - "Target processor architecture is not specified. " -@@ -201,12 +212,12 @@ IF(CPUINFO_SUPPORTED_PLATFORM) - src/arm/linux/chipset.c - src/arm/linux/midr.c - src/arm/linux/hwcap.c) -- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv[5-8]") -+ IF(CPUINFO_TARGET_PROCESSOR MATCHES "^armv[5-8]") - LIST(APPEND CPUINFO_SRCS src/arm/linux/aarch32-isa.c) - IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND ANDROID_ABI STREQUAL "armeabi") - SET_SOURCE_FILES_PROPERTIES(src/arm/linux/aarch32-isa.c PROPERTIES COMPILE_FLAGS -marm) - ENDIF() -- ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") -+ ELSEIF(CPUINFO_TARGET_PROCESSOR MATCHES "^(aarch64|arm64)$") - LIST(APPEND CPUINFO_SRCS src/arm/linux/aarch64-isa.c) - ENDIF() - ELSEIF(IS_APPLE_OS AND CPUINFO_TARGET_PROCESSOR MATCHES "arm64.*") -@@ -395,7 +406,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) - TARGET_COMPILE_DEFINITIONS(cpuinfo_mock PRIVATE _GNU_SOURCE=1) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv5te|armv7-a)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv5te|armv7-a)$") - ADD_EXECUTABLE(atm7029b-tablet-test test/mock/atm7029b-tablet.cc) - TARGET_INCLUDE_DIRECTORIES(atm7029b-tablet-test BEFORE PRIVATE test/mock) - TARGET_LINK_LIBRARIES(atm7029b-tablet-test PRIVATE cpuinfo_mock gtest) -@@ -577,7 +588,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) - ADD_TEST(NAME xperia-sl-test COMMAND xperia-sl-test) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv5te|armv7-a|aarch64)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv5te|armv7-a|aarch64)$") - ADD_EXECUTABLE(alcatel-revvl-test test/mock/alcatel-revvl.cc) - TARGET_INCLUDE_DIRECTORIES(alcatel-revvl-test BEFORE PRIVATE test/mock) - TARGET_LINK_LIBRARIES(alcatel-revvl-test PRIVATE cpuinfo_mock gtest) -@@ -774,7 +785,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) - ADD_TEST(NAME xperia-c4-dual-test COMMAND xperia-c4-dual-test) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(i686|x86_64)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(i686|x86_64)$") - ADD_EXECUTABLE(alldocube-iwork8-test test/mock/alldocube-iwork8.cc) - TARGET_INCLUDE_DIRECTORIES(alldocube-iwork8-test BEFORE PRIVATE test/mock) - TARGET_LINK_LIBRARIES(alldocube-iwork8-test PRIVATE cpuinfo_mock gtest) -@@ -831,7 +842,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_UNIT_TESTS) - ADD_TEST(NAME brand-string-test COMMAND brand-string-test) - ENDIF() - -- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") -+ IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") - ADD_LIBRARY(android_properties_interface STATIC test/name/android-properties-interface.c) - CPUINFO_TARGET_ENABLE_C99(android_properties_interface) - CPUINFO_TARGET_RUNTIME_LIBRARY(android_properties_interface) -@@ -879,7 +890,7 @@ IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_TOOLS) - TARGET_LINK_LIBRARIES(cache-info PRIVATE cpuinfo) - INSTALL(TARGETS cache-info RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - -- IF(CMAKE_SYSTEM_NAME MATCHES "^(Android|Linux)$" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") -+ IF(CMAKE_SYSTEM_NAME MATCHES "^(Android|Linux)$" AND CPUINFO_TARGET_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") - ADD_EXECUTABLE(auxv-dump tools/auxv-dump.c) - CPUINFO_TARGET_ENABLE_C99(auxv-dump) - CPUINFO_TARGET_RUNTIME_LIBRARY(auxv-dump) diff --git a/cmake/vcpkg-ports/cpuinfo/portfile.cmake b/cmake/vcpkg-ports/cpuinfo/portfile.cmake index eeb0007195ca3..3fcf76b7adafc 100644 --- a/cmake/vcpkg-ports/cpuinfo/portfile.cmake +++ b/cmake/vcpkg-ports/cpuinfo/portfile.cmake @@ -11,7 +11,6 @@ vcpkg_from_github( HEAD_REF master PATCHES patch_cpuinfo_h_for_arm64ec.patch - patch_vcpkg_arm64ec_support.patch # https://github.com/pytorch/cpuinfo/pull/324 ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs index 3c92400715740..8cca2b42e987a 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs @@ -368,88 +368,6 @@ public struct OrtApi public IntPtr EpDevice_Device; public IntPtr GetEpApi; public IntPtr GetTensorSizeInBytes; - - public IntPtr AllocatorGetStats; - - public IntPtr CreateMemoryInfo_V2; - public IntPtr MemoryInfoGetDeviceMemType; - public IntPtr MemoryInfoGetVendorId; - - public IntPtr ValueInfo_GetValueProducer; - public IntPtr ValueInfo_GetValueNumConsumers; - public IntPtr ValueInfo_GetValueConsumers; - public IntPtr ValueInfo_GetInitializerValue; - public IntPtr ValueInfo_GetExternalInitializerInfo; - public IntPtr ValueInfo_IsRequiredGraphInput; - public IntPtr ValueInfo_IsOptionalGraphInput; - public IntPtr ValueInfo_IsGraphOutput; - public IntPtr ValueInfo_IsConstantInitializer; - public IntPtr ValueInfo_IsFromOuterScope; - public IntPtr Graph_GetName; - public IntPtr Graph_GetModelPath; - public IntPtr Graph_GetOnnxIRVersion; - public IntPtr Graph_GetNumOperatorSets; - public IntPtr Graph_GetOperatorSets; - public IntPtr Graph_GetNumInputs; - public IntPtr Graph_GetInputs; - public IntPtr Graph_GetNumOutputs; - public IntPtr Graph_GetOutputs; - public IntPtr Graph_GetNumInitializers; - public IntPtr Graph_GetInitializers; - public IntPtr Graph_GetNumNodes; - public IntPtr Graph_GetNodes; - public IntPtr Graph_GetParentNode; - public IntPtr Graph_GetGraphView; - public IntPtr Node_GetId; - public IntPtr Node_GetName; - public IntPtr Node_GetOperatorType; - public IntPtr Node_GetDomain; - public IntPtr Node_GetSinceVersion; - public IntPtr Node_GetNumInputs; - public IntPtr Node_GetInputs; - public IntPtr Node_GetNumOutputs; - public IntPtr Node_GetOutputs; - public IntPtr Node_GetNumImplicitInputs; - public IntPtr Node_GetImplicitInputs; - public IntPtr Node_GetNumAttributes; - public IntPtr Node_GetAttributes; - public IntPtr Node_GetAttributeByName; - public IntPtr Node_GetTensorAttributeAsOrtValue; - public IntPtr OpAttr_GetType; - public IntPtr OpAttr_GetName; - public IntPtr Node_GetNumSubgraphs; - public IntPtr Node_GetSubgraphs; - public IntPtr Node_GetGraph; - public IntPtr Node_GetEpName; - public IntPtr ReleaseExternalInitializerInfo; - public IntPtr ExternalInitializerInfo_GetFilePath; - public IntPtr ExternalInitializerInfo_GetFileOffset; - public IntPtr ExternalInitializerInfo_GetByteSize; - - public IntPtr GetRunConfigEntry; - - public IntPtr EpDevice_MemoryInfo; - - public IntPtr CreateSharedAllocator; - public IntPtr GetSharedAllocator; - public IntPtr ReleaseSharedAllocator; - - public IntPtr GetTensorData; - - public IntPtr GetSessionOptionsConfigEntries; - - public IntPtr SessionGetMemoryInfoForInputs; - public IntPtr SessionGetMemoryInfoForOutputs; - public IntPtr SessionGetEpDeviceForInputs; - - public IntPtr CreateSyncStreamForEpDevice; - public IntPtr SyncStream_GetHandle; - public IntPtr ReleaseSyncStream; - - public IntPtr CopyTensors; - - public IntPtr Graph_GetModelMetadata; - public IntPtr GetModelCompatibilityForEpDevices; } internal static class NativeMethods @@ -786,10 +704,6 @@ static NativeMethods() (DSessionOptionsSetEpSelectionPolicyDelegate)Marshal.GetDelegateForFunctionPointer( api_.SessionOptionsSetEpSelectionPolicyDelegate, typeof(DSessionOptionsSetEpSelectionPolicyDelegate)); - - OrtGetModelCompatibilityForEpDevices = (DOrtGetModelCompatibilityForEpDevices)Marshal.GetDelegateForFunctionPointer( - api_.GetModelCompatibilityForEpDevices, - typeof(DOrtGetModelCompatibilityForEpDevices)); } internal class NativeLib @@ -2542,18 +2456,6 @@ public delegate void DOrtRemoveKeyValuePair(IntPtr /* OrtKeyValuePairs* */ kvps, public static DOrtGetEpDevices OrtGetEpDevices; - /// - /// Validate compiled model compatibility for the provided EP devices. - /// - [UnmanagedFunctionPointer(CallingConvention.Winapi)] - public delegate IntPtr /* OrtStatus* */ DOrtGetModelCompatibilityForEpDevices( - IntPtr[] /* const OrtEpDevice* const* */ ep_devices, - UIntPtr /* size_t */ num_ep_devices, - byte[] /* const char* */ compatibility_info, - out int /* OrtCompiledModelCompatibility */ out_status); - - public static DOrtGetModelCompatibilityForEpDevices OrtGetModelCompatibilityForEpDevices; - /// /// Add execution provider devices to the session options. /// Priority is based on the order of the OrtEpDevice instances. Highest priority first. diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs index 052d5899b52c0..5c70808b82be1 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs @@ -7,21 +7,6 @@ namespace Microsoft.ML.OnnxRuntime { - /// - /// Represents the compatibility status of a pre-compiled model with one or more execution provider devices. - /// - /// - /// This enum is used to determine whether a pre-compiled model can be used with specific execution providers - /// and devices, or if recompilation is needed. - /// - public enum OrtCompiledModelCompatibility - { - EP_NOT_APPLICABLE = 0, - EP_SUPPORTED_OPTIMAL = 1, - EP_SUPPORTED_PREFER_RECOMPILATION = 2, - EP_UNSUPPORTED = 3, - } - /// /// Delegate for logging function callback. /// Supply your function and register it with the environment to receive logging callbacks via @@ -376,31 +361,6 @@ public string[] GetAvailableProviders() } } - /// - /// Validate a compiled model's compatibility information for one or more EP devices. - /// - /// The list of EP devices to validate against. - /// The compatibility string from the precompiled model to validate. - /// OrtCompiledModelCompatibility enum value denoting the compatibility status - public OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( - IReadOnlyList epDevices, string compatibilityInfo) - { - if (epDevices == null || epDevices.Count == 0) - throw new ArgumentException("epDevices must be non-empty", nameof(epDevices)); - - var devicePtrs = new IntPtr[epDevices.Count]; - for (int i = 0; i < epDevices.Count; ++i) - { - devicePtrs[i] = epDevices[i].Handle; - } - - var infoUtf8 = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(compatibilityInfo); - NativeApiStatus.VerifySuccess( - NativeMethods.OrtGetModelCompatibilityForEpDevices( - devicePtrs, (UIntPtr)devicePtrs.Length, infoUtf8, out int status)); - return (OrtCompiledModelCompatibility)status; - } - /// /// Get/Set log level property of OrtEnv instance diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/EpCompatibilityTests.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/EpCompatibilityTests.cs deleted file mode 100644 index 103fe5bc10106..0000000000000 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/EpCompatibilityTests.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// not supported on mobile platforms -#if !(ANDROID || IOS) - -namespace Microsoft.ML.OnnxRuntime.Tests; - -using System; -using System.Linq; -using Xunit; -using System.Collections.Generic; - -public class EpCompatibilityTests -{ - private readonly OrtEnv ortEnvInstance = OrtEnv.Instance(); - - private IReadOnlyList GetDevices() - { - var epDevices = ortEnvInstance.GetEpDevices(); - Assert.NotNull(epDevices); - Assert.NotEmpty(epDevices); - return epDevices; - } - - [Fact] - public void GetEpCompatibility_InvalidArgs() - { - Assert.Throws(() => ortEnvInstance.GetModelCompatibilityForEpDevices(null, "info")); - Assert.Throws(() => ortEnvInstance.GetModelCompatibilityForEpDevices(new List(), "info")); - } - - [Fact] - public void GetEpCompatibility_SingleDeviceCpuProvider() - { - var devices = GetDevices(); - var someInfo = "arbitrary-compat-string"; - - // Use CPU device - var cpu = devices.First(d => d.EpName == "CPUExecutionProvider"); - Assert.NotNull(cpu); - var selected = new List { cpu }; - var status = ortEnvInstance.GetModelCompatibilityForEpDevices(selected, someInfo); - - // CPU defaults to not applicable in this scenario - Assert.Equal(OrtCompiledModelCompatibility.EP_NOT_APPLICABLE, status); - } -} -#endif diff --git a/include/onnxruntime/core/providers/cann/cann_provider_options.h b/include/onnxruntime/core/providers/cann/cann_provider_options.h index 4b33ee77a892e..51b423e68110a 100644 --- a/include/onnxruntime/core/providers/cann/cann_provider_options.h +++ b/include/onnxruntime/core/providers/cann/cann_provider_options.h @@ -15,8 +15,6 @@ struct OrtCANNProviderOptions { onnxruntime::ArenaExtendStrategy arena_extend_strategy; // Strategy used to grow the memory arena int enable_cann_graph; // Flag indicating if prioritizing the use of // CANN's graph-running capabilities - int enable_cann_subgraph; // Flag indicating whether to generate subgraph - // automaticly int dump_graphs; // Flag indicating if dumping graphs int dump_om_model; // Flag indicating if dumping om model std::string precision_mode; // Operator Precision Mode diff --git a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h index 28ce4439fdc7e..21aa797ce16eb 100644 --- a/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h +++ b/include/onnxruntime/core/providers/utils/ort_graph_to_proto.h @@ -232,7 +232,7 @@ static Ort::Status GetOrtValueInfoTensorTypeShape(const OrtValueInfo& ort_value_ /*out*/ std::vector& dims, /*out*/ std::vector& symbolic_dims); static Ort::Status OrtValueInfoToProto(const OrtValueInfo& ort_value_info, onnx::ValueInfoProto& value_info_proto); -static Ort::Status OrtOpAttrToProto(const OrtOpAttr& ort_attr, onnx::AttributeProto& attr_proto); +static Ort::Status OrtOpAttrToProto(const OrtNode& ort_node, const OrtOpAttr& ort_attr, onnx::AttributeProto& attr_proto); Ort::Status OrtGraphToProto(const OrtGraph& ort_graph, onnx::GraphProto& graph_proto, @@ -379,7 +379,7 @@ Ort::Status OrtGraphToProto(const OrtGraph& ort_graph, } onnx::AttributeProto* attr_proto = node_proto->add_attribute(); - ORT_EP_UTILS_CXX_RETURN_IF_ERROR(OrtOpAttrToProto(*ort_attr, *attr_proto)); + ORT_EP_UTILS_CXX_RETURN_IF_ERROR(OrtOpAttrToProto(*ort_node, *ort_attr, *attr_proto)); } } @@ -652,7 +652,7 @@ static Ort::Status OrtValueInfoToProto(const OrtValueInfo& ort_value_info, return Ort::Status{nullptr}; } -static Ort::Status OrtOpAttrToProto(const OrtOpAttr& ort_attr, onnx::AttributeProto& attr_proto) { +static Ort::Status OrtOpAttrToProto(const OrtNode& ort_node, const OrtOpAttr& ort_attr, onnx::AttributeProto& attr_proto) { const OrtApi& ort_api = Ort::GetApi(); const char* attr_name = nullptr; @@ -766,7 +766,7 @@ static Ort::Status OrtOpAttrToProto(const OrtOpAttr& ort_attr, onnx::AttributePr // TensorProto as an attribute value doesn't require a name. OrtValue* ort_value = nullptr; - ORT_EP_UTILS_C_RETURN_IF_ERROR(ort_api.OpAttr_GetTensorAttributeAsOrtValue(&ort_attr, &ort_value)); + ORT_EP_UTILS_C_RETURN_IF_ERROR(ort_api.Node_GetTensorAttributeAsOrtValue(&ort_node, &ort_attr, &ort_value)); Ort::Value tensor(ort_value); diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index f137d88e5fb8a..9ae6174817b7c 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -6079,6 +6079,7 @@ struct OrtApi { /** \brief Get the OrtNode's 'TENSOR' attribute as an OrtValue. * + * \param[in] node The OrtNode instance. * \param[in] attribute The OrtOpAttr instance. * \param[out] attr_tensor If successful, contains the 'TENSOR' attribute as a newly created OrtValue. Must be freed with OrtApi::ReleaseValue. @@ -6087,7 +6088,7 @@ struct OrtApi { * * \since Version 1.23. */ - ORT_API2_STATUS(OpAttr_GetTensorAttributeAsOrtValue, _In_ const OrtOpAttr* attribute, + ORT_API2_STATUS(Node_GetTensorAttributeAsOrtValue, _In_ const OrtNode* node, _In_ const OrtOpAttr* attribute, _Outptr_result_maybenull_ OrtValue** attr_tensor); /** \brief Get the attribute type as OrtOpAttrType from an OrtOpAttr. diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 13675ab447ab1..c39e27088e8bc 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -79,19 +79,22 @@ struct Exception : std::exception { throw Ort::Exception(string, code) #endif +// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, +// it's in a template so that we can define a global variable in a header and make +// it transparent to the users of the API. +template +struct Global { + static const OrtApi* api_; +}; + +// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it. +template #ifdef ORT_API_MANUAL_INIT -// If the macro ORT_API_MANUAL_INIT is defined, no static initialization -// will be performed. Instead, users must call InitApi() before using the -// ORT C++ APIs.. -// -// InitApi() sets the global API object using the default initialization -// logic. Users call this to initialize the ORT C++ APIs at a time that -// makes sense in their program. -inline void InitApi() noexcept; - -// InitApi(const OrtApi*) is used by custom operator libraries that are not -// linked to onnxruntime. It sets the global API object, which is required -// by the ORT C++ APIs. +const OrtApi* Global::api_{}; +inline void InitApi() noexcept { Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); } + +// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is +// required by C++ APIs. // // Example mycustomop.cc: // @@ -104,88 +107,22 @@ inline void InitApi() noexcept; // // ... // } // -inline void InitApi(const OrtApi* api) noexcept; -#endif - -namespace detail { -// This is used internally by the C++ API. This class holds the global -// variable that points to the OrtApi. -struct Global { - static const OrtApi* Api(const OrtApi* newValue = nullptr) noexcept { - // This block-level static will be initialized once when this function is - // first executed, delaying the call to DefaultInit() until it is first needed. - // - // When ORT_API_MANUAL_INIT is not defined, DefaultInit() calls - // OrtGetApiBase()->GetApi(), which may result in a shared library being - // loaded. - // - // Using a block-level static instead of a class-level static helps - // avoid issues with static initialization order and dynamic libraries - // loading other dynamic libraries. - // - // This makes it safe to include the C++ API headers in a shared library - // that is delay loaded or delay loads its dependencies. - // - // This DOES NOT make it safe to _use_ arbitrary ORT C++ APIs when - // initializing static members, however. - static const OrtApi* api = DefaultInit(); - - if (newValue) { - api = newValue; - } - - return api; - } - - private: - // Has different definitions based on ORT_API_MANUAL_INIT - static const OrtApi* DefaultInit() noexcept; - -#ifdef ORT_API_MANUAL_INIT - // Public APIs to set the OrtApi* to use. - friend void ::Ort::InitApi() noexcept; - friend void ::Ort::InitApi(const OrtApi*) noexcept; +inline void InitApi(const OrtApi* api) noexcept { Global::api_ = api; } +#else +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(push) +// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers. +// Please define ORT_API_MANUAL_INIT if it conerns you. +#pragma warning(disable : 26426) #endif -}; -} // namespace detail - -#ifdef ORT_API_MANUAL_INIT - -// See comments on declaration above for usage. -inline void InitApi(const OrtApi* api) noexcept { detail::Global::Api(api); } -inline void InitApi() noexcept { InitApi(OrtGetApiBase()->GetApi(ORT_API_VERSION)); } - -#ifdef _MSC_VER -// If you get a linker error about a mismatch here, you are trying to -// link two compilation units that have different definitions for -// ORT_API_MANUAL_INIT together. All compilation units must agree on the -// definition of ORT_API_MANUAL_INIT. -#pragma detect_mismatch("ORT_API_MANUAL_INIT", "enabled") +const OrtApi* Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(pop) #endif - -inline const OrtApi* detail::Global::DefaultInit() noexcept { - // When ORT_API_MANUAL_INIT is defined, there's no default init that can - // be done. - return nullptr; -} - -#else // ORT_API_MANUAL_INIT - -#ifdef _MSC_VER -// If you get a linker error about a mismatch here, you are trying to link -// two compilation units that have different definitions for -// ORT_API_MANUAL_INIT together. All compilation units must agree on the -// definition of ORT_API_MANUAL_INIT. -#pragma detect_mismatch("ORT_API_MANUAL_INIT", "disabled") #endif -inline const OrtApi* detail::Global::DefaultInit() noexcept { - return OrtGetApiBase()->GetApi(ORT_API_VERSION); -} -#endif // ORT_API_MANUAL_INIT - /// This returns a reference to the ORT C API. -inline const OrtApi& GetApi() noexcept { return *detail::Global::Api(); } +inline const OrtApi& GetApi() noexcept { return *Global::api_; } /// /// This function returns the onnxruntime version string @@ -1076,16 +1013,6 @@ struct EpDevice : detail::EpDeviceImpl { ConstKeyValuePairs ep_metadata = {}, ConstKeyValuePairs ep_options = {}); }; -/** \brief Validate a compiled model's compatibility for one or more EP devices. - * - * Throws on error. Returns the resulting compatibility status. - * /// \param ep_devices The EP devices to check compatibility against. - * /// \param compatibility_info The compatibility string from the precompiled model to validate. - */ -OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( - const std::vector& ep_devices, - const char* compatibility_info); - /** \brief The Env (Environment) * * The Env holds the logging state used by all other objects. diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 05c86ae4e0c58..d0089726812a3 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -859,26 +859,6 @@ inline void CustomOpDomain::Add(const OrtCustomOp* op) { ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); } -inline OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( - const std::vector& ep_devices, - const char* compatibility_info) { - if (ep_devices.empty()) { - ORT_CXX_API_THROW("ep_devices is empty", ORT_INVALID_ARGUMENT); - } - - std::vector ptrs; - ptrs.reserve(ep_devices.size()); - for (const auto& d : ep_devices) ptrs.push_back(d); - - OrtCompiledModelCompatibility status = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; - ThrowOnError(GetApi().GetModelCompatibilityForEpDevices( - reinterpret_cast(ptrs.data()), - ptrs.size(), - compatibility_info, - &status)); - return status; -} - inline LoraAdapter LoraAdapter::CreateLoraAdapter(const std::basic_string& adapter_path, OrtAllocator* allocator) { OrtLoraAdapter* p; diff --git a/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h b/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h index bbd6a43bb7a41..672103bedc437 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_device_ep_metadata_keys.h @@ -12,7 +12,4 @@ static const char* const kOrtEpDevice_EpMetadataKey_Version = "version"; // Prefix for execution provider compatibility information stored in model metadata. // Used when generating EP context models to store compatibility strings for each EP. // Full key format: "ep_compatibility_info." -static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compatibility_info."; - -// Key for the execution provider library path (for dynamically loaded EPs) -static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path"; +static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compatibility_info."; \ No newline at end of file diff --git a/java/src/main/java/ai/onnxruntime/OnnxRuntime.java b/java/src/main/java/ai/onnxruntime/OnnxRuntime.java index 3bb61698f5da7..97423ffb37251 100644 --- a/java/src/main/java/ai/onnxruntime/OnnxRuntime.java +++ b/java/src/main/java/ai/onnxruntime/OnnxRuntime.java @@ -42,8 +42,6 @@ final class OnnxRuntime { private static final int ORT_API_VERSION_13 = 13; // Post 1.13 builds of the ORT API private static final int ORT_API_VERSION_14 = 14; - // Post 1.22 builds of the ORT API - private static final int ORT_API_VERSION_23 = 23; // The initial release of the ORT training API. private static final int ORT_TRAINING_API_VERSION_1 = 1; @@ -105,9 +103,6 @@ final class OnnxRuntime { /** The Training API handle. */ static long ortTrainingApiHandle; - /** The Compile API handle. */ - static long ortCompileApiHandle; - /** Is training enabled in the native library */ static boolean trainingEnabled; @@ -181,13 +176,12 @@ static synchronized void init() throws IOException { } load(ONNXRUNTIME_JNI_LIBRARY_NAME); - ortApiHandle = initialiseAPIBase(ORT_API_VERSION_23); + ortApiHandle = initialiseAPIBase(ORT_API_VERSION_14); if (ortApiHandle == 0L) { throw new IllegalStateException( "There is a mismatch between the ORT class files and the ORT native library, and the native library could not be loaded"); } - ortTrainingApiHandle = initialiseTrainingAPIBase(ortApiHandle, ORT_API_VERSION_23); - ortCompileApiHandle = initialiseCompileAPIBase(ortApiHandle); + ortTrainingApiHandle = initialiseTrainingAPIBase(ortApiHandle, ORT_API_VERSION_14); trainingEnabled = ortTrainingApiHandle != 0L; providers = initialiseProviders(ortApiHandle); version = initialiseVersion(); @@ -505,14 +499,6 @@ private static EnumSet initialiseProviders(long ortApiHandle) { */ private static native long initialiseTrainingAPIBase(long apiHandle, int apiVersionNumber); - /** - * Get a reference to the compile API struct. - * - * @param apiHandle The ORT API struct pointer. - * @return A pointer to the compile API struct. - */ - private static native long initialiseCompileAPIBase(long apiHandle); - /** * Gets the array of available providers. * diff --git a/java/src/main/java/ai/onnxruntime/OrtEnvironment.java b/java/src/main/java/ai/onnxruntime/OrtEnvironment.java index 497772baf5357..8382ef06e26e5 100644 --- a/java/src/main/java/ai/onnxruntime/OrtEnvironment.java +++ b/java/src/main/java/ai/onnxruntime/OrtEnvironment.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024 Oracle and/or its affiliates. All rights reserved. * Licensed under the MIT License. */ package ai.onnxruntime; @@ -8,11 +8,7 @@ import ai.onnxruntime.OrtTrainingSession.OrtCheckpointState; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; import java.util.EnumSet; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.logging.Logger; @@ -446,48 +442,6 @@ public static EnumSet getAvailableProviders() { return OnnxRuntime.providers.clone(); } - /** - * Registers an execution provider library with this OrtEnvironment. - * - * @param registrationName The name to register the library with (used to remove it later with - * {@link #unregisterExecutionProviderLibrary(String)}). - * @param libraryPath The path to the library binary on disk. - * @throws OrtException If the library could not be registered. - */ - public void registerExecutionProviderLibrary(String registrationName, String libraryPath) - throws OrtException { - registerExecutionProviderLibrary( - OnnxRuntime.ortApiHandle, nativeHandle, registrationName, libraryPath); - } - - /** - * Unregisters an execution provider library from this OrtEnvironment. - * - * @param registrationName The name the library was registered under. - * @throws OrtException If the library could not be removed. - */ - public void unregisterExecutionProviderLibrary(String registrationName) throws OrtException { - unregisterExecutionProviderLibrary(OnnxRuntime.ortApiHandle, nativeHandle, registrationName); - } - - /** - * Get the list of all execution provider and device combinations that are available. - * - * @see OrtSession.SessionOptions#addExecutionProvider(List, Map) - * @return The list of execution provider and device combinations. - * @throws OrtException If the devices could not be listed. - */ - public List getEpDevices() throws OrtException { - long[] deviceHandles = getEpDevices(OnnxRuntime.ortApiHandle, nativeHandle); - - List devicesList = new ArrayList<>(); - for (long deviceHandle : deviceHandles) { - devicesList.add(new OrtEpDevice(deviceHandle)); - } - - return Collections.unmodifiableList(devicesList); - } - /** * Creates the native object. * @@ -522,40 +476,6 @@ private static native long createHandle( */ private static native long getDefaultAllocator(long apiHandle) throws OrtException; - /** - * Registers the specified execution provider with this OrtEnvironment. - * - * @param apiHandle The API handle. - * @param nativeHandle The OrtEnvironment handle. - * @param registrationName The name of the execution provider. - * @param libraryPath The path to the execution provider binary. - * @throws OrtException If the registration failed. - */ - private static native void registerExecutionProviderLibrary( - long apiHandle, long nativeHandle, String registrationName, String libraryPath) - throws OrtException; - - /** - * Removes the specified execution provider from this OrtEnvironment. - * - * @param apiHandle The API handle. - * @param nativeHandle The OrtEnvironment handle. - * @param registrationName The name of the execution provider. - * @throws OrtException If the removal failed. - */ - private static native void unregisterExecutionProviderLibrary( - long apiHandle, long nativeHandle, String registrationName) throws OrtException; - - /** - * Gets handles for the EP device tuples available in this OrtEnvironment. - * - * @param apiHandle The API handle to use. - * @param nativeHandle The OrtEnvironment handle. - * @return An array of OrtEpDevice handles. - * @throws OrtException If the call failed. - */ - private static native long[] getEpDevices(long apiHandle, long nativeHandle) throws OrtException; - /** * Closes the OrtEnvironment, frees the handle. * diff --git a/java/src/main/java/ai/onnxruntime/OrtEpDevice.java b/java/src/main/java/ai/onnxruntime/OrtEpDevice.java deleted file mode 100644 index f63dec1dbaf83..0000000000000 --- a/java/src/main/java/ai/onnxruntime/OrtEpDevice.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -package ai.onnxruntime; - -import java.util.Map; - -/** A tuple of Execution Provider information and the hardware device. */ -public final class OrtEpDevice { - - private final long nativeHandle; - - private final String epName; - private final String epVendor; - private final Map epMetadata; - private final Map epOptions; - private final OrtHardwareDevice device; - - /** - * Construct an OrtEpDevice tuple from the native pointer. - * - * @param nativeHandle The native pointer. - */ - OrtEpDevice(long nativeHandle) { - this.nativeHandle = nativeHandle; - this.epName = getName(OnnxRuntime.ortApiHandle, nativeHandle); - this.epVendor = getVendor(OnnxRuntime.ortApiHandle, nativeHandle); - String[][] metadata = getMetadata(OnnxRuntime.ortApiHandle, nativeHandle); - this.epMetadata = OrtUtil.convertToMap(metadata); - String[][] options = getOptions(OnnxRuntime.ortApiHandle, nativeHandle); - this.epOptions = OrtUtil.convertToMap(options); - this.device = new OrtHardwareDevice(getDeviceHandle(OnnxRuntime.ortApiHandle, nativeHandle)); - } - - /** - * Return the native pointer. - * - * @return The native pointer. - */ - long getNativeHandle() { - return nativeHandle; - } - - /** - * Gets the EP name. - * - * @return The EP name. - */ - public String getName() { - return epName; - } - - /** - * Gets the vendor name. - * - * @return The vendor name. - */ - public String getVendor() { - return epVendor; - } - - /** - * Gets an unmodifiable view on the EP metadata. - * - * @return The EP metadata. - */ - public Map getMetadata() { - return epMetadata; - } - - /** - * Gets an unmodifiable view on the EP options. - * - * @return The EP options. - */ - public Map getOptions() { - return epOptions; - } - - /** - * Gets the device information. - * - * @return The device information. - */ - public OrtHardwareDevice getDevice() { - return device; - } - - @Override - public String toString() { - return "OrtEpDevice{" - + "epName='" - + epName - + '\'' - + ", epVendor='" - + epVendor - + '\'' - + ", epMetadata=" - + epMetadata - + ", epOptions=" - + epOptions - + ", device=" - + device - + '}'; - } - - private static native String getName(long apiHandle, long nativeHandle); - - private static native String getVendor(long apiHandle, long nativeHandle); - - private static native String[][] getMetadata(long apiHandle, long nativeHandle); - - private static native String[][] getOptions(long apiHandle, long nativeHandle); - - private static native long getDeviceHandle(long apiHandle, long nativeHandle); -} diff --git a/java/src/main/java/ai/onnxruntime/OrtHardwareDevice.java b/java/src/main/java/ai/onnxruntime/OrtHardwareDevice.java deleted file mode 100644 index bd99f5599fd14..0000000000000 --- a/java/src/main/java/ai/onnxruntime/OrtHardwareDevice.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -package ai.onnxruntime; - -import java.util.Map; -import java.util.logging.Logger; - -/** Hardware information for a specific device. */ -public final class OrtHardwareDevice { - - /** The hardware device types. */ - // Must be updated in concert with the native OrtHardwareDeviceType enum in the C API - public enum OrtHardwareDeviceType { - /** A CPU device. */ - CPU(0), - /** A GPU device. */ - GPU(1), - /** A NPU (Neural Processing Unit) device. */ - NPU(2); - private final int value; - - private static final Logger logger = Logger.getLogger(OrtHardwareDeviceType.class.getName()); - private static final OrtHardwareDeviceType[] values = new OrtHardwareDeviceType[3]; - - static { - for (OrtHardwareDeviceType ot : OrtHardwareDeviceType.values()) { - values[ot.value] = ot; - } - } - - OrtHardwareDeviceType(int value) { - this.value = value; - } - - /** - * Gets the native value associated with this device type. - * - * @return The native value. - */ - public int getValue() { - return value; - } - - /** - * Maps from the C API's int enum to the Java enum. - * - * @param deviceType The index of the Java enum. - * @return The Java enum. - */ - public static OrtHardwareDeviceType mapFromInt(int deviceType) { - if ((deviceType >= 0) && (deviceType < values.length)) { - return values[deviceType]; - } else { - logger.warning("Unknown device type '" + deviceType + "' setting to CPU"); - return CPU; - } - } - } - - private final long nativeHandle; - - private final OrtHardwareDeviceType type; - private final int vendorId; - private final String vendor; - private final int deviceId; - private final Map metadata; - - OrtHardwareDevice(long nativeHandle) { - this.nativeHandle = nativeHandle; - this.type = - OrtHardwareDeviceType.mapFromInt(getDeviceType(OnnxRuntime.ortApiHandle, nativeHandle)); - this.vendorId = getVendorId(OnnxRuntime.ortApiHandle, nativeHandle); - this.vendor = getVendor(OnnxRuntime.ortApiHandle, nativeHandle); - this.deviceId = getDeviceId(OnnxRuntime.ortApiHandle, nativeHandle); - String[][] metadata = getMetadata(OnnxRuntime.ortApiHandle, nativeHandle); - this.metadata = OrtUtil.convertToMap(metadata); - } - - long getNativeHandle() { - return nativeHandle; - } - - /** - * Gets the device type. - * - * @return The device type. - */ - public OrtHardwareDeviceType getType() { - return type; - } - - /** - * Gets the vendor ID number. - * - * @return The vendor ID number. - */ - public int getVendorId() { - return vendorId; - } - - /** - * Gets the device ID number. - * - * @return The device ID number. - */ - public int getDeviceId() { - return deviceId; - } - - /** - * Gets an unmodifiable view on the device metadata. - * - * @return The device metadata. - */ - public Map getMetadata() { - return metadata; - } - - /** - * Gets the vendor name. - * - * @return The vendor name. - */ - public String getVendor() { - return vendor; - } - - @Override - public String toString() { - return "OrtHardwareDevice{" - + "type=" - + type - + ", vendorId=" - + vendorId - + ", vendor='" - + vendor - + '\'' - + ", deviceId=" - + deviceId - + ", metadata=" - + metadata - + '}'; - } - - private static native String getVendor(long apiHandle, long nativeHandle); - - private static native String[][] getMetadata(long apiHandle, long nativeHandle); - - private static native int getDeviceType(long apiHandle, long nativeHandle); - - private static native int getDeviceId(long apiHandle, long nativeHandle); - - private static native int getVendorId(long apiHandle, long nativeHandle); -} diff --git a/java/src/main/java/ai/onnxruntime/OrtModelCompilationOptions.java b/java/src/main/java/ai/onnxruntime/OrtModelCompilationOptions.java deleted file mode 100644 index 09b3064b72b93..0000000000000 --- a/java/src/main/java/ai/onnxruntime/OrtModelCompilationOptions.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -package ai.onnxruntime; - -import java.nio.ByteBuffer; -import java.util.EnumSet; - -/** Configuration options for compiling ONNX models. */ -public final class OrtModelCompilationOptions implements AutoCloseable { - /** Flags representing options when compiling a model. */ - public enum OrtCompileApiFlags implements OrtFlags { - /** Default. Do not enable any additional compilation options. */ - NONE(0), - - /** - * Force compilation to return an error (ORT_FAIL) if no nodes were compiled. Otherwise, a model - * with basic optimizations (ORT_ENABLE_BASIC) is still generated by default. - */ - ERROR_IF_NO_NODES_COMPILED(1), - - /** - * Force compilation to return an error (ORT_FAIL) if a file with the same filename as the - * output model exists. Otherwise, compilation will automatically overwrite the output file if - * it exists. - */ - ERROR_IF_OUTPUT_FILE_EXISTS(1 << 1); - - /** The native value of the enum. */ - public final int value; - - OrtCompileApiFlags(int value) { - this.value = value; - } - - @Override - public int getValue() { - return value; - } - } - - private final long nativeHandle; - private boolean closed = false; - - // Used to ensure the byte buffer doesn't get GC'd before the model is compiled. - private ByteBuffer buffer; - - OrtModelCompilationOptions(long nativeHandle) { - this.nativeHandle = nativeHandle; - } - - /** - * Creates a model compilation options from an existing SessionOptions. - * - *

An OrtModelCompilationOptions object contains the settings used to generate a compiled ONNX - * model. The OrtSessionOptions object has the execution providers with which the model will be - * compiled. - * - * @param env The OrtEnvironment. - * @param sessionOptions The session options to use. - * @return A constructed model compilation options instance. - * @throws OrtException If the construction failed. - */ - public static OrtModelCompilationOptions createFromSessionOptions( - OrtEnvironment env, OrtSession.SessionOptions sessionOptions) throws OrtException { - long handle = - createFromSessionOptions( - OnnxRuntime.ortApiHandle, - OnnxRuntime.ortCompileApiHandle, - env.getNativeHandle(), - sessionOptions.getNativeHandle()); - return new OrtModelCompilationOptions(handle); - } - - /** - * Checks if the OrtModelCompilationOptions is closed, if so throws {@link IllegalStateException}. - */ - private void checkClosed() { - if (closed) { - throw new IllegalStateException("Trying to use a closed OrtModelCompilationOptions."); - } - } - - @Override - public void close() { - if (!closed) { - close(OnnxRuntime.ortCompileApiHandle, nativeHandle); - closed = true; - } else { - throw new IllegalStateException("Trying to close a closed OrtModelCompilationOptions."); - } - } - - /** - * Sets the file path to the input ONNX model. - * - *

The input model's location must be set either to a path on disk with this method, or by - * supplying an in-memory reference with {@link #setInputModelFromBuffer}. - * - * @param inputModelPath The path to the model on disk. - * @throws OrtException If the set failed. - */ - public void setInputModelPath(String inputModelPath) throws OrtException { - checkClosed(); - setInputModelPath( - OnnxRuntime.ortApiHandle, OnnxRuntime.ortCompileApiHandle, nativeHandle, inputModelPath); - } - - /** - * Uses the supplied buffer as the input ONNX model. - * - *

The input model's location must be set either to an in-memory reference with this method, or - * by supplying a path on disk with {@link #setInputModelPath(String)}. - * - *

If the {@link ByteBuffer} is not direct it is copied into a direct buffer. In either case - * this object holds a reference to the buffer to prevent it from being GC'd. - * - * @param inputModelBuffer The buffer. - * @throws OrtException If the buffer could not be set. - */ - public void setInputModelFromBuffer(ByteBuffer inputModelBuffer) throws OrtException { - checkClosed(); - if (!inputModelBuffer.isDirect()) { - // if it's not a direct buffer, copy it. - buffer = ByteBuffer.allocateDirect(inputModelBuffer.remaining()); - int tmpPos = inputModelBuffer.position(); - buffer.put(inputModelBuffer); - buffer.rewind(); - inputModelBuffer.position(tmpPos); - } else { - buffer = inputModelBuffer; - } - int bufferPos = buffer.position(); - int bufferRemaining = buffer.remaining(); - setInputModelFromBuffer( - OnnxRuntime.ortApiHandle, - OnnxRuntime.ortCompileApiHandle, - nativeHandle, - buffer, - bufferPos, - bufferRemaining); - } - - /** - * Sets the file path for the output compiled ONNX model. - * - *

If this is unset it will append `_ctx` to the file name, e.g., my_model.onnx becomes - * my_model_ctx.onnx. - * - * @param outputModelPath The output model path. - * @throws OrtException If the path could not be set. - */ - public void setOutputModelPath(String outputModelPath) throws OrtException { - checkClosed(); - setOutputModelPath( - OnnxRuntime.ortApiHandle, OnnxRuntime.ortCompileApiHandle, nativeHandle, outputModelPath); - } - - /** - * Optionally sets the file that stores initializers for the compiled ONNX model. If unset then - * initializers are stored inside the model. - * - *

Only initializers for nodes that were not compiled are stored in the external initializers - * file. Compiled nodes contain their initializer data within the `ep_cache_context` attribute of - * EPContext nodes. - * - * @see OrtModelCompilationOptions#setEpContextEmbedMode - * @param outputExternalInitializersPath Path to the file. - * @param sizeThreshold Initializers larger than this threshold are stored in the file. - * @throws OrtException If the path could not be set. - */ - public void setOutputExternalInitializersPath( - String outputExternalInitializersPath, long sizeThreshold) throws OrtException { - checkClosed(); - // check positive - setOutputExternalInitializersPath( - OnnxRuntime.ortApiHandle, - OnnxRuntime.ortCompileApiHandle, - nativeHandle, - outputExternalInitializersPath, - sizeThreshold); - } - - /** - * Enables or disables the embedding of EPContext binary data into the ep_cache_context attribute - * of EPContext nodes. - * - *

Defaults to false. When enabled, the `ep_cache_context` attribute of EPContext nodes will - * store the context binary data, which may include weights for compiled subgraphs. When disabled, - * the `ep_cache_context` attribute of EPContext nodes will contain the path to the file - * containing the context binary data. The path is set by the execution provider creating the - * EPContext node. - * - *

For more details see the EPContext design - * document. - * - * @param embedEpContext True to embed EPContext binary data into the EPContext node's - * ep_cache_context attribute. - * @throws OrtException If the set operation failed. - */ - public void setEpContextEmbedMode(boolean embedEpContext) throws OrtException { - checkClosed(); - setEpContextEmbedMode( - OnnxRuntime.ortApiHandle, OnnxRuntime.ortCompileApiHandle, nativeHandle, embedEpContext); - } - - /** - * Sets the specified compilation flags. - * - * @param flags The compilation flags. - * @throws OrtException If the set operation failed. - */ - public void setCompilationFlags(EnumSet flags) throws OrtException { - checkClosed(); - setCompilationFlags( - OnnxRuntime.ortApiHandle, - OnnxRuntime.ortCompileApiHandle, - nativeHandle, - OrtFlags.aggregateToInt(flags)); - } - - /** - * Compiles the ONNX model with the configuration described by this instance of - * OrtModelCompilationOptions. - * - * @throws OrtException If the compilation failed. - */ - public void compileModel() throws OrtException { - checkClosed(); - // Safe as the environment must exist to create one of these objects. - OrtEnvironment env = OrtEnvironment.getEnvironment(); - compileModel( - OnnxRuntime.ortApiHandle, - OnnxRuntime.ortCompileApiHandle, - env.getNativeHandle(), - nativeHandle); - } - - private static native long createFromSessionOptions( - long apiHandle, long compileApiHandle, long envHandle, long nativeHandle) throws OrtException; - - private static native void close(long compileApiHandle, long nativeHandle); - - private static native void setInputModelPath( - long apiHandle, long compileApiHandle, long nativeHandle, String inputModelPath) - throws OrtException; - - private static native void setInputModelFromBuffer( - long apiHandle, - long compileApiHandle, - long nativeHandle, - ByteBuffer inputBuffer, - long bufferPos, - long bufferRemaining) - throws OrtException; - - private static native void setOutputModelPath( - long apiHandle, long compileApiHandle, long nativeHandle, String outputModelPath) - throws OrtException; - - private static native void setOutputExternalInitializersPath( - long apiHandle, - long compileApiHandle, - long nativeHandle, - String externalInitializersPath, - long sizeThreshold) - throws OrtException; - - private static native void setEpContextEmbedMode( - long apiHandle, long compileApiHandle, long nativeHandle, boolean embedEpContext) - throws OrtException; - - private static native void setCompilationFlags( - long apiHandle, long compileApiHandle, long nativeHandle, int flags) throws OrtException; - - private static native void compileModel( - long apiHandle, long compileApiHandle, long envHandle, long nativeHandle) throws OrtException; -} diff --git a/java/src/main/java/ai/onnxruntime/OrtSession.java b/java/src/main/java/ai/onnxruntime/OrtSession.java index 42dc90b71cb80..a399d5080ca16 100644 --- a/java/src/main/java/ai/onnxruntime/OrtSession.java +++ b/java/src/main/java/ai/onnxruntime/OrtSession.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates * Licensed under the MIT License. */ @@ -8,6 +8,7 @@ import ai.onnxruntime.providers.CoreMLFlags; import ai.onnxruntime.providers.NNAPIFlags; import ai.onnxruntime.providers.OrtCUDAProviderOptions; +import ai.onnxruntime.providers.OrtFlags; import ai.onnxruntime.providers.OrtTensorRTProviderOptions; import java.io.IOException; import java.nio.ByteBuffer; @@ -623,10 +624,6 @@ private native OnnxModelMetadata constructMetadata( *

Used to set the number of threads, optimisation level, computation backend and other * options. * - *

The order execution providers are added to an options instance is the order they will be - * considered for op node assignment, with the EP added first having priority. The CPU EP is a - * fallback and added by default. - * *

Modifying this after the session has been constructed will have no effect. * *

The SessionOptions object must not be closed until all sessions which use it are closed, as @@ -733,7 +730,7 @@ public SessionOptions() { @Override public void close() { if (!closed) { - if (!customLibraryHandles.isEmpty()) { + if (customLibraryHandles.size() > 0) { long[] longArray = new long[customLibraryHandles.size()]; for (int i = 0; i < customLibraryHandles.size(); i++) { longArray[i] = customLibraryHandles.get(i); @@ -920,10 +917,10 @@ public void registerCustomOpLibrary(String path) throws OrtException { * *

 OrtStatus* (*fn)(OrtSessionOptions* options, const OrtApiBase* api); * - *

See Add - * Custom Op for more information on custom ops. See an example of a custom op library - * registration function here. + *

See https://onnxruntime.ai/docs/reference/operators/add-custom-op.html for more + * information on custom ops. See + * https://github.com/microsoft/onnxruntime/blob/342a5bf2b756d1a1fc6fdc582cfeac15182632fe/onnxruntime/test/testdata/custom_op_library/custom_op_library.cc#L115 + * for an example of a custom op library registration function. * * @param registrationFuncName The name of the registration function to call. * @throws OrtException If there was an error finding or calling the registration function. @@ -1276,47 +1273,10 @@ public void addCoreML(EnumSet flags) throws OrtException { addCoreML(OnnxRuntime.ortApiHandle, nativeHandle, OrtFlags.aggregateToInt(flags)); } - /** - * Adds the specified execution provider and device tuples as an execution backend. - * - *

Execution provider priority is in the order added, i.e., the first provider added to a - * session options will be used first for op node assignment. - * - * @param devices The EP and device tuples. Each element must use the same EP, though they can - * use different devices. - * @param providerOptions Configuration options for the execution provider. Refer to the - * specific execution provider's documentation. - * @throws OrtException If there was an error in native code. - */ - public void addExecutionProvider(List devices, Map providerOptions) - throws OrtException { - checkClosed(); - if (devices.isEmpty()) { - throw new IllegalArgumentException("Must supply at least one OrtEpDevice"); - } - long[] deviceHandles = new long[devices.size()]; - for (int i = 0; i < devices.size(); i++) { - deviceHandles[i] = devices.get(i).getNativeHandle(); - } - String[][] optsArray = OrtUtil.unpackMap(providerOptions); - // This is valid as the environment must have been created to create the OrtEpDevice list. - long envHandle = OrtEnvironment.getEnvironment().getNativeHandle(); - addExecutionProvider( - OnnxRuntime.ortApiHandle, - envHandle, - nativeHandle, - deviceHandles, - optsArray[0], - optsArray[1]); - } - /** * Adds the named execution provider (backend) as an execution backend. This generic function * only allows a subset of execution providers. * - *

Execution provider priority is in the order added, i.e., the first provider added to a - * session options will be used first for op node assignment. - * * @param providerName The name of the execution provider. * @param providerOptions Configuration options for the execution provider. Refer to the * specific execution provider's documentation. @@ -1325,9 +1285,20 @@ public void addExecutionProvider(List devices, Map private void addExecutionProvider(String providerName, Map providerOptions) throws OrtException { checkClosed(); - String[][] optsArray = OrtUtil.unpackMap(providerOptions); + String[] providerOptionKey = new String[providerOptions.size()]; + String[] providerOptionVal = new String[providerOptions.size()]; + int i = 0; + for (Map.Entry entry : providerOptions.entrySet()) { + providerOptionKey[i] = entry.getKey(); + providerOptionVal[i] = entry.getValue(); + i++; + } addExecutionProvider( - OnnxRuntime.ortApiHandle, nativeHandle, providerName, optsArray[0], optsArray[1]); + OnnxRuntime.ortApiHandle, + nativeHandle, + providerName, + providerOptionKey, + providerOptionVal); } /** @@ -1513,15 +1484,6 @@ private native void addExecutionProvider( String[] providerOptionKey, String[] providerOptionVal) throws OrtException; - - private native void addExecutionProvider( - long apiHandle, - long envHandle, - long nativeHandle, - long[] deviceHandles, - String[] providerOptionKey, - String[] providerOptionVal) - throws OrtException; } /** Used to control logging and termination of a call to {@link OrtSession#run}. */ diff --git a/java/src/main/java/ai/onnxruntime/OrtUtil.java b/java/src/main/java/ai/onnxruntime/OrtUtil.java index ee91fdb292baa..2f44236e4ef67 100644 --- a/java/src/main/java/ai/onnxruntime/OrtUtil.java +++ b/java/src/main/java/ai/onnxruntime/OrtUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. */ @@ -16,9 +16,6 @@ import java.nio.ShortBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.logging.Logger; /** Util code for interacting with Java arrays. */ @@ -373,52 +370,6 @@ public static boolean validateShape(long[] shape) { return valid && shape.length <= TensorInfo.MAX_DIMENSIONS; } - /** - * Converts the output of a OrtKeyValuePairs into a Java unmodifiable HashMap. - * - * @param zippedString The zipped keys and values. - * @return An unmodifiable Map. - */ - static Map convertToMap(String[][] zippedString) { - if (zippedString.length != 2) { - throw new IllegalArgumentException("Invalid zipped string, must have two arrays."); - } else if (zippedString[0].length != zippedString[1].length) { - throw new IllegalArgumentException( - "Invalid zipped string, must have two arrays of the same length."); - } - Map map = new HashMap<>(capacityFromSize(zippedString[0].length)); - for (int i = 0; i < zippedString[0].length; i++) { - map.put(zippedString[0][i], zippedString[1][i]); - } - return Collections.unmodifiableMap(map); - } - - /** - * Converts a Java string map into a pair of arrays suitable for constructing a native - * OrtKeyValuePairs object. - * - * @param map A map from string to string, with no null keys or values. - * @return A pair of String arrays. - */ - static String[][] unpackMap(Map map) { - String[] keys = new String[map.size()]; - String[] values = new String[map.size()]; - int i = 0; - for (Map.Entry entry : map.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - throw new IllegalArgumentException( - "Invalid map, keys and values must not be null, found key = " - + entry.getKey() - + ", value = " - + entry.getValue()); - } - keys[i] = entry.getKey(); - values[i] = entry.getValue(); - i++; - } - return new String[][] {keys, values}; - } - /** * Flatten a multidimensional String array into a single dimensional String array, reading it in a * multidimensional row-major order. diff --git a/java/src/main/java/ai/onnxruntime/providers/CoreMLFlags.java b/java/src/main/java/ai/onnxruntime/providers/CoreMLFlags.java index 15fe459dad7c8..22bf940844774 100644 --- a/java/src/main/java/ai/onnxruntime/providers/CoreMLFlags.java +++ b/java/src/main/java/ai/onnxruntime/providers/CoreMLFlags.java @@ -1,11 +1,9 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * Licensed under the MIT License. */ package ai.onnxruntime.providers; -import ai.onnxruntime.OrtFlags; - /** Flags for the CoreML provider. */ public enum CoreMLFlags implements OrtFlags { /** diff --git a/java/src/main/java/ai/onnxruntime/providers/NNAPIFlags.java b/java/src/main/java/ai/onnxruntime/providers/NNAPIFlags.java index dd30684078717..eeaf6cc8d53bc 100644 --- a/java/src/main/java/ai/onnxruntime/providers/NNAPIFlags.java +++ b/java/src/main/java/ai/onnxruntime/providers/NNAPIFlags.java @@ -1,11 +1,9 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. * Licensed under the MIT License. */ package ai.onnxruntime.providers; -import ai.onnxruntime.OrtFlags; - /** Flags for the NNAPI provider. */ public enum NNAPIFlags implements OrtFlags { /** Enables fp16 support. */ diff --git a/java/src/main/java/ai/onnxruntime/OrtFlags.java b/java/src/main/java/ai/onnxruntime/providers/OrtFlags.java similarity index 88% rename from java/src/main/java/ai/onnxruntime/OrtFlags.java rename to java/src/main/java/ai/onnxruntime/providers/OrtFlags.java index f57fd945dbeec..73d3eeae6499c 100644 --- a/java/src/main/java/ai/onnxruntime/OrtFlags.java +++ b/java/src/main/java/ai/onnxruntime/providers/OrtFlags.java @@ -1,8 +1,8 @@ /* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. * Licensed under the MIT License. */ -package ai.onnxruntime; +package ai.onnxruntime.providers; import java.util.EnumSet; diff --git a/java/src/main/native/OrtJniUtil.c b/java/src/main/native/OrtJniUtil.c index 96ea8e79bc978..5d8efd7b476cb 100644 --- a/java/src/main/native/OrtJniUtil.c +++ b/java/src/main/native/OrtJniUtil.c @@ -1014,36 +1014,6 @@ jobject convertOrtValueToONNXValue(JNIEnv *jniEnv, const OrtApi * api, OrtAlloca } } -jobjectArray convertOrtKeyValuePairsToArrays(JNIEnv *jniEnv, const OrtApi * api, const OrtKeyValuePairs * kvp) { - // extract pair arrays - const char* const* keys = NULL; - const char* const* values = NULL; - size_t numKeys = 0; - api->GetKeyValuePairs(kvp, &keys, &values, &numKeys); - jsize jNumKeys = safecast_size_t_to_jsize(numKeys); - - // create Java String[] - jclass stringClazz = (*jniEnv)->FindClass(jniEnv, "java/lang/String"); - jobjectArray keyArray = (*jniEnv)->NewObjectArray(jniEnv, jNumKeys, stringClazz, NULL); - jobjectArray valueArray = (*jniEnv)->NewObjectArray(jniEnv, jNumKeys, stringClazz, NULL); - - // populate Java arrays - for (jsize i = 0; i < jNumKeys; i++) { - jstring key = (*jniEnv)->NewStringUTF(jniEnv, keys[i]); - (*jniEnv)->SetObjectArrayElement(jniEnv, keyArray, i, key); - jstring value = (*jniEnv)->NewStringUTF(jniEnv, values[i]); - (*jniEnv)->SetObjectArrayElement(jniEnv, valueArray, i, value); - } - - // create Java String[][] - jclass stringArrClazz = (*jniEnv)->GetObjectClass(jniEnv, keyArray); - jobjectArray pair = (*jniEnv)->NewObjectArray(jniEnv, 2, stringArrClazz, 0); - (*jniEnv)->SetObjectArrayElement(jniEnv, pair, 0, keyArray); - (*jniEnv)->SetObjectArrayElement(jniEnv, pair, 1, valueArray); - - return pair; -} - jint throwOrtException(JNIEnv *jniEnv, int messageId, const char *message) { jstring messageStr = (*jniEnv)->NewStringUTF(jniEnv, message); diff --git a/java/src/main/native/OrtJniUtil.h b/java/src/main/native/OrtJniUtil.h index 040fd41264c10..7f41e06371f2a 100644 --- a/java/src/main/native/OrtJniUtil.h +++ b/java/src/main/native/OrtJniUtil.h @@ -78,8 +78,6 @@ jobject createMapInfoFromValue(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator jobject convertOrtValueToONNXValue(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* onnxValue); -jobjectArray convertOrtKeyValuePairsToArrays(JNIEnv *jniEnv, const OrtApi * api, const OrtKeyValuePairs * kvp); - jint throwOrtException(JNIEnv *env, int messageId, const char *message); jint convertErrorCode(OrtErrorCode code); diff --git a/java/src/main/native/ai_onnxruntime_OnnxRuntime.c b/java/src/main/native/ai_onnxruntime_OnnxRuntime.c index d8f5f1a3cb2db..659f34e1fb66f 100644 --- a/java/src/main/native/ai_onnxruntime_OnnxRuntime.c +++ b/java/src/main/native/ai_onnxruntime_OnnxRuntime.c @@ -32,19 +32,6 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OnnxRuntime_initialiseTrainingAPIBas return (jlong) trainingApi; } -/* - * Class: ai_onnxruntime_OnnxRuntime - * Method: initialiseCompileAPIBase - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OnnxRuntime_initialiseCompileAPIBase - (JNIEnv * jniEnv, jclass clazz, jlong apiHandle) { - (void)jniEnv; (void)clazz; // required JNI parameters not needed by functions which don't call back into Java. - const OrtApi* api = (const OrtApi*)apiHandle; - const OrtCompileApi* compileApi = api->GetCompileApi(); - return (jlong) compileApi; -} - /* * Class: ai_onnxruntime_OnnxRuntime * Method: getAvailableProviders diff --git a/java/src/main/native/ai_onnxruntime_OrtEnvironment.c b/java/src/main/native/ai_onnxruntime_OrtEnvironment.c index 77b096d62ec76..e1b1ff1c05fe1 100644 --- a/java/src/main/native/ai_onnxruntime_OrtEnvironment.c +++ b/java/src/main/native/ai_onnxruntime_OrtEnvironment.c @@ -60,76 +60,6 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtEnvironment_getDefaultAllocator return (jlong)allocator; } -/* - * Class: ai_onnxruntime_OrtEnvironment - * Method: registerExecutionProviderLibrary - * Signature: (JJLjava/lang/String;Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtEnvironment_registerExecutionProviderLibrary - (JNIEnv * jniEnv, jclass jobj, jlong apiHandle, jlong nativeHandle, jstring name, jstring libraryPath) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEnv* env = (OrtEnv*) nativeHandle; - const char* cName = (*jniEnv)->GetStringUTFChars(jniEnv, name, NULL); -#ifdef _WIN32 - const jchar* cPath = (*jniEnv)->GetStringChars(jniEnv, libraryPath, NULL); - size_t stringLength = (*jniEnv)->GetStringLength(jniEnv, libraryPath); - wchar_t* newString = (wchar_t*)calloc(stringLength + 1, sizeof(wchar_t)); - if (newString == NULL) { - (*jniEnv)->ReleaseStringChars(jniEnv, libraryPath, cPath); - throwOrtException(jniEnv, 1, "Not enough memory"); - return; - } - wcsncpy_s(newString, stringLength + 1, (const wchar_t*)cPath, stringLength); - checkOrtStatus(jniEnv, api, api->RegisterExecutionProviderLibrary(env, cName, newString)); - free(newString); - (*jniEnv)->ReleaseStringChars(jniEnv, libraryPath, cPath); -#else - const char* cPath = (*jniEnv)->GetStringUTFChars(jniEnv, libraryPath, NULL); - checkOrtStatus(jniEnv, api, api->RegisterExecutionProviderLibrary(env, cName, cPath)); - (*jniEnv)->ReleaseStringUTFChars(jniEnv, libraryPath, cPath); -#endif - (*jniEnv)->ReleaseStringUTFChars(jniEnv, name, cName); -} - -/* - * Class: ai_onnxruntime_OrtEnvironment - * Method: unregisterExecutionProviderLibrary - * Signature: (JJLjava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtEnvironment_unregisterExecutionProviderLibrary - (JNIEnv * jniEnv, jclass jobj, jlong apiHandle, jlong nativeHandle, jstring name) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEnv* env = (OrtEnv*) nativeHandle; - const char* cName = (*jniEnv)->GetStringUTFChars(jniEnv, name, NULL); - checkOrtStatus(jniEnv, api, api->UnregisterExecutionProviderLibrary(env, cName)); - (*jniEnv)->ReleaseStringUTFChars(jniEnv, name, cName); -} - -/* - * Class: ai_onnxruntime_OrtEnvironment - * Method: getEpDevices - * Signature: (JJ)[J - */ -JNIEXPORT jlongArray JNICALL Java_ai_onnxruntime_OrtEnvironment_getEpDevices - (JNIEnv * jniEnv, jclass jobj, jlong apiHandle, jlong nativeHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEnv* env = (OrtEnv*) nativeHandle; - size_t numDevices = 0; - const OrtEpDevice* const* devicesArr = NULL; - OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetEpDevices(env, &devicesArr, &numDevices)); - if (code != ORT_OK) { - return NULL; - } else { - jsize numDevicesInt = safecast_size_t_to_jsize(numDevices); - jlongArray outputArr = (*jniEnv)->NewLongArray(jniEnv, numDevicesInt); - (*jniEnv)->SetLongArrayRegion(jniEnv, outputArr, 0, numDevicesInt, (jlong*)devicesArr); - return outputArr; - } -} - /* * Class: ai_onnxruntime_OrtEnvironment * Method: close diff --git a/java/src/main/native/ai_onnxruntime_OrtEpDevice.c b/java/src/main/native/ai_onnxruntime_OrtEpDevice.c deleted file mode 100644 index 5a1e3092b0fb9..0000000000000 --- a/java/src/main/native/ai_onnxruntime_OrtEpDevice.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -#include -#include "onnxruntime/core/session/onnxruntime_c_api.h" -#include "OrtJniUtil.h" -#include "ai_onnxruntime_OrtEpDevice.h" - -/* - * Class: ai_onnxruntime_OrtEpDevice - * Method: getName - * Signature: (JJ)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtEpDevice_getName - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEpDevice* epDevice = (OrtEpDevice*) nativeHandle; - const char* name = api->EpDevice_EpName(epDevice); - jstring nameStr = (*jniEnv)->NewStringUTF(jniEnv, name); - return nameStr; -} - -/* - * Class: ai_onnxruntime_OrtEpDevice - * Method: getVendor - * Signature: (JJ)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtEpDevice_getVendor - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEpDevice* epDevice = (OrtEpDevice*) nativeHandle; - const char* vendor = api->EpDevice_EpVendor(epDevice); - jstring vendorStr = (*jniEnv)->NewStringUTF(jniEnv, vendor); - return vendorStr; -} - -/* - * Class: ai_onnxruntime_OrtEpDevice - * Method: getMetadata - * Signature: (JJ)[[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtEpDevice_getMetadata - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEpDevice* epDevice = (OrtEpDevice*) nativeHandle; - const OrtKeyValuePairs* kvp = api->EpDevice_EpMetadata(epDevice); - jobjectArray pair = convertOrtKeyValuePairsToArrays(jniEnv, api, kvp); - return pair; -} - -/* - * Class: ai_onnxruntime_OrtEpDevice - * Method: getOptions - * Signature: (JJ)[[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtEpDevice_getOptions - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEpDevice* epDevice = (OrtEpDevice*) nativeHandle; - const OrtKeyValuePairs* kvp = api->EpDevice_EpOptions(epDevice); - jobjectArray pair = convertOrtKeyValuePairsToArrays(jniEnv, api, kvp); - return pair; -} - -/* - * Class: ai_onnxruntime_OrtEpDevice - * Method: getDeviceHandle - * Signature: (JJ)J - */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtEpDevice_getDeviceHandle - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jniEnv; (void) jclazz; // Required JNI parameters not needed by functions which don't need to access their host object or the JVM. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtEpDevice* epDevice = (OrtEpDevice*) nativeHandle; - const OrtHardwareDevice* device = api->EpDevice_Device(epDevice); - return (jlong) device; -} diff --git a/java/src/main/native/ai_onnxruntime_OrtHardwareDevice.c b/java/src/main/native/ai_onnxruntime_OrtHardwareDevice.c deleted file mode 100644 index 3191a89c26ba1..0000000000000 --- a/java/src/main/native/ai_onnxruntime_OrtHardwareDevice.c +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -#include -#include "onnxruntime/core/session/onnxruntime_c_api.h" -#include "OrtJniUtil.h" -#include "ai_onnxruntime_OrtHardwareDevice.h" - -/* - * Class: ai_onnxruntime_OrtHardwareDevice - * Method: getVendor - * Signature: (JJ)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtHardwareDevice_getVendor - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtHardwareDevice* device = (OrtHardwareDevice*) nativeHandle; - const char* vendor = api->HardwareDevice_Vendor(device); - jstring vendorStr = (*jniEnv)->NewStringUTF(jniEnv, vendor); - return vendorStr; -} - -/* - * Class: ai_onnxruntime_OrtHardwareDevice - * Method: getMetadata - * Signature: (JJ)[[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtHardwareDevice_getMetadata - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtHardwareDevice* device = (OrtHardwareDevice*) nativeHandle; - const OrtKeyValuePairs* kvp = api->HardwareDevice_Metadata(device); - jobjectArray pair = convertOrtKeyValuePairsToArrays(jniEnv, api, kvp); - return pair; -} - -/* - * Class: ai_onnxruntime_OrtHardwareDevice - * Method: getDeviceType - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_ai_onnxruntime_OrtHardwareDevice_getDeviceType - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtHardwareDevice* device = (OrtHardwareDevice*) nativeHandle; - OrtHardwareDeviceType type = api->HardwareDevice_Type(device); - jint output = 0; - // Must be kept aligned with the Java OrtHardwareDeviceType enum. - switch (type) { - case OrtHardwareDeviceType_CPU: - output = 0; - break; - case OrtHardwareDeviceType_GPU: - output = 1; - break; - case OrtHardwareDeviceType_NPU: - output = 2; - break; - default: - throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "Unexpected device type found. Only CPU, GPU and NPU are supported."); - break; - } - return output; -} - -/* - * Class: ai_onnxruntime_OrtHardwareDevice - * Method: getDeviceId - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_ai_onnxruntime_OrtHardwareDevice_getDeviceId - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jniEnv; (void) jclazz; // Required JNI parameters not needed by functions which don't need to access their host object or the JVM. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtHardwareDevice* device = (OrtHardwareDevice*) nativeHandle; - uint32_t id = api->HardwareDevice_DeviceId(device); - return (jint) id; -} - -/* - * Class: ai_onnxruntime_OrtHardwareDevice - * Method: getVendorId - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_ai_onnxruntime_OrtHardwareDevice_getVendorId - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong nativeHandle) { - (void) jniEnv; (void) jclazz; // Required JNI parameters not needed by functions which don't need to access their host object or the JVM. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtHardwareDevice* device = (OrtHardwareDevice*) nativeHandle; - uint32_t id = api->HardwareDevice_VendorId(device); - return (jint) id; -} diff --git a/java/src/main/native/ai_onnxruntime_OrtModelCompilationOptions.c b/java/src/main/native/ai_onnxruntime_OrtModelCompilationOptions.c deleted file mode 100644 index 4f79383d09766..0000000000000 --- a/java/src/main/native/ai_onnxruntime_OrtModelCompilationOptions.c +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -#include -#include "onnxruntime/core/session/onnxruntime_c_api.h" -#include "OrtJniUtil.h" -#include "ai_onnxruntime_OrtModelCompilationOptions.h" - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: createFromSessionOptions - * Signature: (JJJJ)J - */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_createFromSessionOptions - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong envHandle, jlong sessionOptionsHandle) { - (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*)apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*)compileApiHandle; - const OrtEnv* env = (const OrtEnv*)envHandle; - const OrtSessionOptions* sessionOptions = (const OrtSessionOptions*) sessionOptionsHandle; - OrtModelCompilationOptions* output = NULL; - checkOrtStatus(jniEnv, api, compileApi->CreateModelCompilationOptionsFromSessionOptions(env, sessionOptions, &output)); - return (jlong) output; -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: close - * Signature: (JJ)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_close - (JNIEnv * jniEnv, jclass jclazz, jlong compileApiHandle, jlong nativeHandle) { - (void)jniEnv; (void)jclazz; // Required JNI parameters not needed by functions which don't need to access their host object or the JVM. - const OrtCompileApi* compileApi = (const OrtCompileApi*)compileApiHandle; - compileApi->ReleaseModelCompilationOptions((OrtModelCompilationOptions *)nativeHandle); -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: setInputModelPath - * Signature: (JJJLjava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_setInputModelPath - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong nativeHandle, jstring modelPath) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*) compileApiHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; -#ifdef _WIN32 - const jchar* cPath = (*jniEnv)->GetStringChars(jniEnv, modelPath, NULL); - size_t stringLength = (*jniEnv)->GetStringLength(jniEnv, modelPath); - wchar_t* newString = (wchar_t*)calloc(stringLength + 1, sizeof(wchar_t)); - if (newString == NULL) { - (*jniEnv)->ReleaseStringChars(jniEnv, modelPath, cPath); - throwOrtException(jniEnv, 1, "Not enough memory"); - return; - } - wcsncpy_s(newString, stringLength + 1, (const wchar_t*)cPath, stringLength); - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetInputModelPath(compOpts, newString)); - free(newString); - (*jniEnv)->ReleaseStringChars(jniEnv, modelPath, cPath); -#else - const char* cPath = (*jniEnv)->GetStringUTFChars(jniEnv, modelPath, NULL); - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetInputModelPath(compOpts, cPath)); - (*jniEnv)->ReleaseStringUTFChars(jniEnv, modelPath, cPath); -#endif -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: setInputModelFromBuffer - * Signature: (JJJLjava/nio/ByteBuffer;JJ)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_setInputModelFromBuffer - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong nativeHandle, jobject buffer, jlong bufferPos, jlong bufferRemaining) { - (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - // Cast to pointers - const OrtApi* api = (const OrtApi*)apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*)compileApiHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; - - // Extract the buffer - char* bufferArr = (char*)(*jniEnv)->GetDirectBufferAddress(jniEnv, buffer); - // Increment by bufferPos bytes - bufferArr = bufferArr + bufferPos; - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetInputModelFromBuffer(compOpts, bufferArr, bufferRemaining)); -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: setOutputModelPath - * Signature: (JJJLjava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_setOutputModelPath - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong nativeHandle, jstring outputPath) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*) compileApiHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; -#ifdef _WIN32 - const jchar* cPath = (*jniEnv)->GetStringChars(jniEnv, outputPath, NULL); - size_t stringLength = (*jniEnv)->GetStringLength(jniEnv, outputPath); - wchar_t* newString = (wchar_t*)calloc(stringLength + 1, sizeof(wchar_t)); - if (newString == NULL) { - (*jniEnv)->ReleaseStringChars(jniEnv, outputPath, cPath); - throwOrtException(jniEnv, 1, "Not enough memory"); - return; - } - wcsncpy_s(newString, stringLength + 1, (const wchar_t*)cPath, stringLength); - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetOutputModelPath(compOpts, newString)); - free(newString); - (*jniEnv)->ReleaseStringChars(jniEnv, outputPath, cPath); -#else - const char* cPath = (*jniEnv)->GetStringUTFChars(jniEnv, outputPath, NULL); - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetOutputModelPath(compOpts, cPath)); - (*jniEnv)->ReleaseStringUTFChars(jniEnv, outputPath, cPath); -#endif -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: setOutputExternalInitializersPath - * Signature: (JJJLjava/lang/String;J)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_setOutputExternalInitializersPath - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong nativeHandle, jstring initializersPath, jlong threshold) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*) compileApiHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; -#ifdef _WIN32 - const jchar* cPath = (*jniEnv)->GetStringChars(jniEnv, initializersPath, NULL); - size_t stringLength = (*jniEnv)->GetStringLength(jniEnv, initializersPath); - wchar_t* newString = (wchar_t*)calloc(stringLength + 1, sizeof(wchar_t)); - if (newString == NULL) { - (*jniEnv)->ReleaseStringChars(jniEnv, initializersPath, cPath); - throwOrtException(jniEnv, 1, "Not enough memory"); - return; - } - wcsncpy_s(newString, stringLength + 1, (const wchar_t*)cPath, stringLength); - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetOutputModelExternalInitializersFile(compOpts, newString, threshold)); - free(newString); - (*jniEnv)->ReleaseStringChars(jniEnv, initializersPath, cPath); -#else - const char* cPath = (*jniEnv)->GetStringUTFChars(jniEnv, initializersPath, NULL); - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetOutputModelExternalInitializersFile(compOpts, cPath, threshold)); - (*jniEnv)->ReleaseStringUTFChars(jniEnv, initializersPath, cPath); -#endif -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: setEpContextEmbedMode - * Signature: (JJJZ)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_setEpContextEmbedMode - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong nativeHandle, jboolean embedMode) { - (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*)apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*)compileApiHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetEpContextEmbedMode(compOpts, (bool) embedMode)); -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: setCompilationFlags - * Signature: (JJJI)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_setCompilationFlags - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong nativeHandle, jint flags) { - (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*)apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*)compileApiHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; - checkOrtStatus(jniEnv, api, compileApi->ModelCompilationOptions_SetFlags(compOpts, flags)); -} - -/* - * Class: ai_onnxruntime_OrtModelCompilationOptions - * Method: compileModel - * Signature: (JJJJ)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtModelCompilationOptions_compileModel - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong compileApiHandle, jlong envHandle, jlong nativeHandle) { - (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*)apiHandle; - const OrtCompileApi* compileApi = (const OrtCompileApi*)compileApiHandle; - const OrtEnv* env = (const OrtEnv*)envHandle; - OrtModelCompilationOptions* compOpts = (OrtModelCompilationOptions *) nativeHandle; - checkOrtStatus(jniEnv, api, compileApi->CompileModel(env, compOpts)); -} diff --git a/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c b/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c index 95bcdf7af9746..ff6b7fa703e6e 100644 --- a/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c +++ b/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c @@ -718,11 +718,11 @@ JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addROC } /* - * Class: ai_onnxruntime_OrtSession_SessionOptions + * Class:: ai_onnxruntime_OrtSession_SessionOptions * Method: addExecutionProvider - * Signature: (JJLjava/lang/String;[Ljava/lang/String;[Ljava/lang/String;)V + * Signature: (JILjava/lang/String)V */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addExecutionProvider__JJLjava_lang_String_2_3Ljava_lang_String_2_3Ljava_lang_String_2( +JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addExecutionProvider( JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong optionsHandle, jstring jepName, jobjectArray configKeyArr, jobjectArray configValueArr) { (void)jobj; @@ -756,50 +756,3 @@ JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addExe free((void*)jkeyArray); free((void*)jvalueArray); } - -/* - * Class: ai_onnxruntime_OrtSession_SessionOptions - * Method: addExecutionProvider - * Signature: (JJJ[J[Ljava/lang/String;[Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addExecutionProvider__JJJ_3J_3Ljava_lang_String_2_3Ljava_lang_String_2 - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong envHandle, jlong optionsHandle, jlongArray deviceHandleArr, jobjectArray configKeyArr, jobjectArray configValueArr) { - (void)jobj; - - const OrtApi* api = (const OrtApi*)apiHandle; - OrtEnv* env = (OrtEnv*) envHandle; - OrtSessionOptions* options = (OrtSessionOptions*)optionsHandle; - jsize deviceCount = (*jniEnv)->GetArrayLength(jniEnv, deviceHandleArr); - jsize keyCount = (*jniEnv)->GetArrayLength(jniEnv, configKeyArr); - - const char** keyArray = (const char**)allocarray(keyCount, sizeof(const char*)); - const char** valueArray = (const char**)allocarray(keyCount, sizeof(const char*)); - jstring* jkeyArray = (jstring*)allocarray(keyCount, sizeof(jstring)); - jstring* jvalueArray = (jstring*)allocarray(keyCount, sizeof(jstring)); - const OrtEpDevice** devicePtrs = allocarray(deviceCount, sizeof(OrtEpDevice *)); - - jlong* deviceHandleElements = (*jniEnv)->GetLongArrayElements(jniEnv, deviceHandleArr, NULL); - for (jsize i = 0; i < deviceCount; i++) { - devicePtrs[i] = (OrtEpDevice*) deviceHandleElements[i]; - } - (*jniEnv)->ReleaseLongArrayElements(jniEnv, deviceHandleArr, deviceHandleElements, JNI_ABORT); - - for (jsize i = 0; i < keyCount; i++) { - jkeyArray[i] = (jstring)((*jniEnv)->GetObjectArrayElement(jniEnv, configKeyArr, i)); - jvalueArray[i] = (jstring)((*jniEnv)->GetObjectArrayElement(jniEnv, configValueArr, i)); - keyArray[i] = (*jniEnv)->GetStringUTFChars(jniEnv, jkeyArray[i], NULL); - valueArray[i] = (*jniEnv)->GetStringUTFChars(jniEnv, jvalueArray[i], NULL); - } - - checkOrtStatus(jniEnv, api, api->SessionOptionsAppendExecutionProvider_V2(options, env, devicePtrs, deviceCount, keyArray, valueArray, keyCount)); - - for (jsize i = 0; i < keyCount; i++) { - (*jniEnv)->ReleaseStringUTFChars(jniEnv, jkeyArray[i], keyArray[i]); - (*jniEnv)->ReleaseStringUTFChars(jniEnv, jvalueArray[i], valueArray[i]); - } - free((void*)devicePtrs); - free((void*)keyArray); - free((void*)valueArray); - free((void*)jkeyArray); - free((void*)jvalueArray); -} diff --git a/java/src/test/java/ai/onnxruntime/CompileApiTest.java b/java/src/test/java/ai/onnxruntime/CompileApiTest.java deleted file mode 100644 index b70f4dca5cbd0..0000000000000 --- a/java/src/test/java/ai/onnxruntime/CompileApiTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -package ai.onnxruntime; - -import ai.onnxruntime.OrtSession.SessionOptions; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** Test for the compilation API. */ -public class CompileApiTest { - private final OrtEnvironment env = OrtEnvironment.getEnvironment(); - - @Test - public void basicUsage() throws OrtException, IOException { - SessionOptions so = new SessionOptions(); - try (OrtModelCompilationOptions compileOptions = - OrtModelCompilationOptions.createFromSessionOptions(env, so)) { - // mainly checking these don't throw which ensures all the plumbing for the binding works. - compileOptions.setInputModelPath("model.onnx"); - compileOptions.setOutputModelPath("compiled_model.onnx"); - - compileOptions.setOutputExternalInitializersPath("external_data.bin", 512); - compileOptions.setEpContextEmbedMode(true); - } - - try (OrtModelCompilationOptions compileOptions = - OrtModelCompilationOptions.createFromSessionOptions(env, so)) { - Path modelPath = TestHelpers.getResourcePath("/squeezenet.onnx"); - byte[] modelBytes = Files.readAllBytes(modelPath); - ByteBuffer modelBuffer = ByteBuffer.wrap(modelBytes); - compileOptions.setInputModelFromBuffer(modelBuffer); - compileOptions.setOutputModelPath("compiled_model.onnx"); - - File f = new File("compiled_model.onnx"); - - compileOptions.compileModel(); - - // Check the compiled model is valid - try (OrtSession session = env.createSession(f.toString(), so)) { - Assertions.assertNotNull(session); - } - - f.delete(); - } - } -} diff --git a/java/src/test/java/ai/onnxruntime/EpDeviceTest.java b/java/src/test/java/ai/onnxruntime/EpDeviceTest.java deleted file mode 100644 index ec4c977508c8c..0000000000000 --- a/java/src/test/java/ai/onnxruntime/EpDeviceTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * Licensed under the MIT License. - */ -package ai.onnxruntime; - -import ai.onnxruntime.OrtHardwareDevice.OrtHardwareDeviceType; -import ai.onnxruntime.OrtSession.SessionOptions; -import java.io.File; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledOnOs; -import org.junit.jupiter.api.condition.OS; - -/** Tests for {@link OrtEpDevice} and {@link OrtHardwareDevice}. */ -@EnabledOnOs(value = OS.WINDOWS) -public class EpDeviceTest { - private final OrtEnvironment ortEnv = OrtEnvironment.getEnvironment(); - - private void readHardwareDeviceValues(OrtHardwareDevice device) { - OrtHardwareDeviceType type = device.getType(); - - Assertions.assertTrue( - type == OrtHardwareDeviceType.CPU - || type == OrtHardwareDeviceType.GPU - || type == OrtHardwareDeviceType.NPU); - - if (type == OrtHardwareDeviceType.CPU) { - Assertions.assertFalse(device.getVendor().isEmpty()); - } else { - Assertions.assertTrue(device.getVendorId() != 0); - Assertions.assertTrue(device.getDeviceId() != 0); - } - - Map metadata = device.getMetadata(); - Assertions.assertNotNull(metadata); - for (Map.Entry kvp : metadata.entrySet()) { - Assertions.assertFalse(kvp.getKey().isEmpty()); - } - } - - @Test - public void getEpDevices() throws OrtException { - List epDevices = ortEnv.getEpDevices(); - Assertions.assertNotNull(epDevices); - Assertions.assertFalse(epDevices.isEmpty()); - for (OrtEpDevice epDevice : epDevices) { - Assertions.assertFalse(epDevice.getName().isEmpty()); - Assertions.assertFalse(epDevice.getVendor().isEmpty()); - Map metadata = epDevice.getMetadata(); - Assertions.assertNotNull(metadata); - Map options = epDevice.getOptions(); - Assertions.assertNotNull(options); - readHardwareDeviceValues(epDevice.getDevice()); - } - } - - @Test - public void registerUnregisterLibrary() throws OrtException { - String libFullPath = TestHelpers.getResourcePath("/example_plugin_ep.dll").toString(); - Assertions.assertTrue( - new File(libFullPath).exists(), "Expected lib " + libFullPath + " does not exist."); - - // example plugin ep uses the registration name as the ep name - String epName = "java_ep"; - - // register. shouldn't throw - ortEnv.registerExecutionProviderLibrary(epName, libFullPath); - - // check OrtEpDevice was found - List epDevices = ortEnv.getEpDevices(); - boolean found = epDevices.stream().anyMatch(a -> a.getName().equals(epName)); - Assertions.assertTrue(found); - - // unregister - ortEnv.unregisterExecutionProviderLibrary(epName); - } - - @Test - public void appendToSessionOptionsV2() { - Consumer>> runTest = - (Supplier> options) -> { - try (SessionOptions sessionOptions = new SessionOptions()) { - sessionOptions.setSessionLogLevel(OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE); - - List epDevices = ortEnv.getEpDevices(); - - // cpu ep ignores the provider options so we can use any value in epOptions and it won't - // break. - List selectedEpDevices = - epDevices.stream() - .filter(a -> a.getName().equals("CPUExecutionProvider")) - .collect(Collectors.toList()); - - Map epOptions = options.get(); - sessionOptions.addExecutionProvider(selectedEpDevices, epOptions); - - Path model = TestHelpers.getResourcePath("/squeezenet.onnx"); - String modelPath = model.toString(); - - // session should load successfully - try (OrtSession session = ortEnv.createSession(modelPath, sessionOptions)) { - Assertions.assertNotNull(session); - } - } catch (OrtException e) { - throw new RuntimeException(e); - } - }; - - // empty options - runTest.accept(Collections::emptyMap); - - // dummy options - runTest.accept(() -> Collections.singletonMap("random_key", "value")); - } -} diff --git a/js/node/src/inference_session_wrap.cc b/js/node/src/inference_session_wrap.cc index 8db91f792cb06..84ed3457a488b 100644 --- a/js/node/src/inference_session_wrap.cc +++ b/js/node/src/inference_session_wrap.cc @@ -15,7 +15,7 @@ Napi::Object InferenceSessionWrap::Init(Napi::Env env, Napi::Object exports) { // create ONNX runtime env Ort::InitApi(); ORT_NAPI_THROW_ERROR_IF( - &Ort::GetApi() == nullptr, env, + Ort::Global::api_ == nullptr, env, "Failed to initialize ONNX Runtime API. It could happen when this nodejs binding was built with a higher version " "ONNX Runtime but now runs with a lower version ONNX Runtime DLL(or shared library)."); diff --git a/onnxruntime/contrib_ops/cpu/bert/gqa_attention_base.h b/onnxruntime/contrib_ops/cpu/bert/gqa_attention_base.h index bfa450f4287f8..0d5117709c18a 100644 --- a/onnxruntime/contrib_ops/cpu/bert/gqa_attention_base.h +++ b/onnxruntime/contrib_ops/cpu/bert/gqa_attention_base.h @@ -280,18 +280,6 @@ class GQAAttentionBase { output, static_cast(present_buffer_sequence_length), nullptr); } - // Pre-allocate buffer for attention mask to avoid allocating it for every processed token - float* attention_bias_thread_fp32 = nullptr; - if (attention_bias_thread != nullptr) { - if constexpr (!std::is_same_v) { - static_assert(std::is_same_v && std::is_same_v); - - size_t bytes = attention_total_seqlen * sizeof(float); - attention_bias_thread_fp32 = static_cast(allocator->Alloc(bytes)); - } - } - BufferUniquePtr scratch_buffer(attention_bias_thread_fp32, BufferDeleter(allocator)); - // compute Softmax U* output_softmax = output; for (size_t seq = 0; seq < sequence_length; seq++) { @@ -328,6 +316,9 @@ class GQAAttentionBase { static_cast(window_size)); } else { static_assert(std::is_same_v && std::is_same_v); + size_t bytes = window_size * sizeof(float); + auto attention_bias_thread_fp32 = static_cast(allocator->Alloc(bytes)); + BufferUniquePtr scratch_buffer(attention_bias_thread_fp32, BufferDeleter(allocator)); MlasConvertHalfToFloatBuffer(attention_bias_thread + start_offset, attention_bias_thread_fp32, window_size); ApplyAttentionBias(output_softmax + start_offset, attention_bias_thread_fp32, static_cast(window_size)); diff --git a/onnxruntime/contrib_ops/cpu/moe/moe_quantization_cpu.cc b/onnxruntime/contrib_ops/cpu/moe/moe_quantization_cpu.cc index 5c6c3b919b572..9b35a40f64f2a 100644 --- a/onnxruntime/contrib_ops/cpu/moe/moe_quantization_cpu.cc +++ b/onnxruntime/contrib_ops/cpu/moe/moe_quantization_cpu.cc @@ -331,13 +331,7 @@ Status QMoECPU::Compute(OpKernelContext* context) const { const int64_t token_idx = route_idx / k_; const float weight = route_scale[route_idx]; - const size_t buffer_offset = static_cast(token_idx) * static_cast(hidden_size); - if (buffer_offset + static_cast(hidden_size) > output_buffer_size) { - // Skip this token to prevent buffer overflow - continue; - } - - float* dest = thread_local_outputs + static_cast(thread_id) * output_buffer_size + buffer_offset; + float* dest = thread_local_outputs + static_cast(thread_id) * output_buffer_size + token_idx * hidden_size; const float* src = C2 + i * hidden_size; for (int64_t j = 0; j < hidden_size; ++j) { dest[j] += weight * (src[j] + (B2_bias ? bias2_float[j] : 0.0f)); @@ -350,9 +344,8 @@ Status QMoECPU::Compute(OpKernelContext* context) const { auto accumulate = [&](float* buffer) { memset(buffer, 0, output_buffer_size * sizeof(float)); for (int i = 0; i < num_expert_threads; ++i) { - const size_t thread_offset = static_cast(i) * output_buffer_size; for (size_t j = 0; j < output_buffer_size; ++j) { - buffer[j] += thread_local_outputs[thread_offset + j]; + buffer[j] += thread_local_outputs[static_cast(i) * output_buffer_size + j]; } } }; diff --git a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc index 36a6f70cc69d9..85a2cbaea0e44 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc @@ -200,19 +200,6 @@ class DynamicQuantizeMatMul final : public MatMulIntegerToFloatBase { can_use_dynamic_quant_mlas_ = (!b_quantization_might_be_asymmetric && b_scale_available); - // Kleidi dynamic path requires strictly positive, finite scales. - // Disable if any invalid scale is detected. - if (can_use_dynamic_quant_mlas_) { - const auto bs = b_scale_tensor->DataAsSpan(); - const bool has_invalid = - std::any_of(bs.begin(), bs.end(), - [](float s) { return !std::isfinite(s) || s <= 0.0f; }); - - if (has_invalid) { - can_use_dynamic_quant_mlas_ = false; - } - } - // Currently, MlasDynamicQGemmBatch() and associated functions require SME or else they are no-ops. // We check that here too before attempting to use them. if (!CPUIDInfo::GetCPUIDInfo().HasArm_SME()) { @@ -392,7 +379,7 @@ Status DynamicQuantizeMatMul::Compute(OpKernelContext* ctx) const { if (y->Shape().Size() == 0) return Status::OK(); - const float* a_data = ctx->Input(IN_A)->Data(); + auto a_data = static_cast(ctx->Input(IN_A)->DataRaw()); auto* y_data = y->MutableData(); // batch gemm @@ -406,7 +393,7 @@ Status DynamicQuantizeMatMul::Compute(OpKernelContext* ctx) const { for (size_t gemm_idx = 0; gemm_idx < num_gemms; gemm_idx++) { auto& params = gemm_data_vec[gemm_idx]; - params.A = a_data + helper.LeftOffsets()[gemm_idx]; + params.A = reinterpret_cast(a_data + helper.LeftOffsets()[gemm_idx]); params.lda = gemm_shape.K; params.PackedB = packed_b_.get(); params.C = y_data + helper.OutputOffsets()[gemm_idx]; diff --git a/onnxruntime/core/common/cpuid_arch_definition.h b/onnxruntime/core/common/cpuid_arch_definition.h index 5946b8ca27067..a541eb66d8ba3 100644 --- a/onnxruntime/core/common/cpuid_arch_definition.h +++ b/onnxruntime/core/common/cpuid_arch_definition.h @@ -9,6 +9,6 @@ #define CPUIDINFO_ARCH_X86 #endif -#if defined(_M_ARM64) || defined(_M_ARM64EC) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__) +#if defined(_M_ARM64) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__) #define CPUIDINFO_ARCH_ARM #endif // ARM or ARM64 diff --git a/onnxruntime/core/graph/abi_graph_types.h b/onnxruntime/core/graph/abi_graph_types.h index 2ef7c4a9091f3..b99c22edb36c8 100644 --- a/onnxruntime/core/graph/abi_graph_types.h +++ b/onnxruntime/core/graph/abi_graph_types.h @@ -252,6 +252,16 @@ struct OrtNode { /// A status indicating success or an error. virtual onnxruntime::Status GetAttributes(gsl::span attrs) const = 0; + ///

+ /// Gets the node's 'TENSOR' attribute as an OrtValue. + /// + /// Node's 'TENSOR' attribute. + /// Output parameter is set to a newly created OrtValue containing the 'TENSOR' attribute value, + /// only if the attribute is of type 'TENSOR' + /// A status indicating success or an error. + virtual onnxruntime::Status GetTensorAttributeAsOrtValue(const OrtOpAttr* attr, + OrtValue*& value) const = 0; + /// /// Gets the number of node subgraphs. /// diff --git a/onnxruntime/core/graph/ep_api_types.cc b/onnxruntime/core/graph/ep_api_types.cc index 92eb31f0ad385..759a2998ace3a 100644 --- a/onnxruntime/core/graph/ep_api_types.cc +++ b/onnxruntime/core/graph/ep_api_types.cc @@ -249,6 +249,32 @@ Status EpNode::GetAttributes(gsl::span dst) const { return Status::OK(); } +Status EpNode::GetTensorAttributeAsOrtValue(const OrtOpAttr* attribute, OrtValue*& result) const { + const auto* attr_proto = reinterpret_cast(attribute); + + if (attr_proto->type() != onnx::AttributeProto::TENSOR) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "This OrtOpAttr instance is not a 'TENSOR' attribute"); + } + + const auto& graph_viewer = ep_graph_->GetGraphViewer(); + const auto& tensor_proto = attr_proto->t(); + + // Check that TensorProto is valid. + ORT_ENFORCE(utils::HasDataType(tensor_proto), "Tensor proto doesn't have data type."); + ORT_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(tensor_proto.data_type()), "Tensor proto has invalid data type."); + ORT_ENFORCE(!utils::HasExternalData(tensor_proto), + "Tensor proto with external data for value attribute is not supported."); + + // Initialize OrtValue for tensor attribute. + auto tensor_attribute_value = std::make_unique(); + AllocatorPtr tensor_attribute_allocator = CPUAllocator::DefaultInstance(); + ORT_RETURN_IF_ERROR(utils::TensorProtoToOrtValue(Env::Default(), graph_viewer.ModelPath(), tensor_proto, + tensor_attribute_allocator, *tensor_attribute_value)); + + result = tensor_attribute_value.release(); + return Status::OK(); +} + Status EpNode::GetNumSubgraphs(size_t& num_subgraphs) const { num_subgraphs = subgraphs_.size(); return Status::OK(); @@ -327,9 +353,6 @@ static Status GetInputIndices(const EpNode& consumer_node, [&found, &value_info_name, &indices](gsl::span input_value_infos, bool is_implicit) -> void { for (size_t i = 0; i < input_value_infos.size(); i++) { - if (input_value_infos[i] == nullptr) { // input_value_info == nullptr means the input is optional - continue; - } if (input_value_infos[i]->GetName() == value_info_name) { indices.push_back(is_implicit ? -1 : static_cast(i)); found = true; diff --git a/onnxruntime/core/graph/ep_api_types.h b/onnxruntime/core/graph/ep_api_types.h index e003f02a79a2d..7f22e265129f7 100644 --- a/onnxruntime/core/graph/ep_api_types.h +++ b/onnxruntime/core/graph/ep_api_types.h @@ -183,6 +183,9 @@ struct EpNode : public OrtNode { // Gets the node's attributes. Status GetAttributes(gsl::span attrs) const override; + Status GetTensorAttributeAsOrtValue(const OrtOpAttr* attribute, + OrtValue*& attr_tensor) const override; + // Gets the number of subgraphs contained by this node. Status GetNumSubgraphs(size_t& num_subgraphs) const override; diff --git a/onnxruntime/core/graph/model_editor_api_types.h b/onnxruntime/core/graph/model_editor_api_types.h index 2c0f6d6174303..e7ffcbc7e4c90 100644 --- a/onnxruntime/core/graph/model_editor_api_types.h +++ b/onnxruntime/core/graph/model_editor_api_types.h @@ -138,6 +138,11 @@ struct ModelEditorNode : public OrtNode { "OrtModelEditorApi does not support getting attribute OrtOpAttr for OrtNode"); } + Status GetTensorAttributeAsOrtValue(const OrtOpAttr* /*attribute*/, OrtValue*& /*attr_tensor*/) const override { + return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, + "OrtModelEditorApi does not support getting 'TENSOR' attribute for OrtNode"); + } + Status GetNumSubgraphs(size_t& /*num_subgraphs*/) const override { return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "OrtModelEditorApi does not support getting the subgraphs for OrtNode"); diff --git a/onnxruntime/core/mlas/lib/kleidiai/sgemm_kleidiai.cpp b/onnxruntime/core/mlas/lib/kleidiai/sgemm_kleidiai.cpp index c579ff1542eb9..caa445b71e2a5 100644 --- a/onnxruntime/core/mlas/lib/kleidiai/sgemm_kleidiai.cpp +++ b/onnxruntime/core/mlas/lib/kleidiai/sgemm_kleidiai.cpp @@ -153,23 +153,28 @@ ArmKleidiAI::MlasGemmBatch( MLAS_THREADPOOL* ThreadPool ) { - if (M == 0 || N == 0) { - return true; + if(TransA == CblasTrans) + { + return false; } - - if (Data->alpha == 0.0f || K == 0) { - if (Data->beta == 0.0f) { - for (size_t i = 0; i < M; ++i) { - std::fill_n(Data->C + i * Data->ldc, N, 0.0f); - } - } else if (Data->beta != 1.0f) { + if (TransA == CblasNoTrans && K == 0) { + if (Data->beta != 1.0f) { for (size_t i = 0; i < M; ++i) { for (size_t j = 0; j < N; ++j) { Data->C[i * Data->ldc + j] *= Data->beta; } } } - return true; + } + if (Data->beta == 0.0f){ + std::fill_n(Data->C, M * Data->ldc, 0.0f); + } + //Fallback in the case of unsupported cases + if (M == 0 || N == 0 || K == 0 || + TransA != CblasNoTrans || + (TransB != CblasNoTrans && !Data[0].BIsPacked)) + { + return false; } if (TransA == CblasNoTrans) { @@ -180,9 +185,11 @@ ArmKleidiAI::MlasGemmBatch( auto m_step = kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa(); auto n_step = kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa(); - if (M < m_step && N < n_step && !Data->BIsPacked) { - // Fallback to MLAS - return false; + if (M < m_step || N < n_step) { + if (GetMlasPlatform().MlasGemmBatchOverride != ArmKleidiAI::MlasGemmBatch){ + //Fallback to MLAS + return false; + } } std::vector KaiPackedData; @@ -309,7 +316,7 @@ ArmKleidiAI::MlasGemmBatch( float* dst_tile = reinterpret_cast(CTile); // quick copy of data in cases where we are not scaling or accumulating anything - // with bounds checking on tile sizing to ensure the data fits in the memory block + // with bounds checking on tile sizing to ensure the data fits in the memory block bool can_memcpy = ( Data[BIdx].alpha == 1.0f && Data[BIdx].beta == 0.0f && @@ -321,37 +328,21 @@ ArmKleidiAI::MlasGemmBatch( if (can_memcpy) { std::memcpy(dst_tile, temp_tile, TileSizeM * TileSizeN * sizeof(float)); - return; - } + }else { + // apply alpha scaling and beta to output files + for (size_t i = 0; i < TileSizeM; ++i) { + for (size_t j = 0; j < TileSizeN; ++j) { + const size_t idx = i * TileSizeN + j; + const size_t dst_idx = i * Data[BIdx].ldc + j; - float alpha = Data[BIdx].alpha; - float beta = Data[BIdx].beta; - size_t ldc = Data[BIdx].ldc; - - for (size_t i = 0; i < TileSizeM; ++i) { - for (size_t j = 0; j < TileSizeN; ++j) { - const size_t temp_idx = i * TileSizeN + j; - const size_t dst_idx = i * ldc + j; - - float ab = temp_tile[temp_idx]; - float c_orig = dst_tile[dst_idx]; - - if (alpha == 1.0f && beta == 0.0f) { - dst_tile[dst_idx] = ab; - } else if (alpha == 1.0f) { - dst_tile[dst_idx] = ab + beta * c_orig; - } else if (beta == 0.0f) { - dst_tile[dst_idx] = alpha * ab; - } else { - dst_tile[dst_idx] = alpha * ab + beta * c_orig; + float ab = temp_tile[idx]; + float c_orig = dst_tile[dst_idx]; + + dst_tile[dst_idx] = Data[BIdx].alpha * ab + Data[BIdx].beta * c_orig; } } } - return; }); - return true; - } - else { - return false; } + return true; } diff --git a/onnxruntime/core/providers/cann/cann_execution_provider.cc b/onnxruntime/core/providers/cann/cann_execution_provider.cc index 06c3628eb301d..4bcf71335d15e 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider.cc @@ -1266,16 +1266,17 @@ CANNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewe // the single operator operation mode of CANN if (info_.enable_cann_graph) { std::vector&& unsupported_nodes = SupportONNXModel(graph_viewer); - if (info_.enable_cann_subgraph && !unsupported_nodes.empty()) { + + if (unsupported_nodes.empty()) { + auto sub_graph = GetSubGraph(graph_viewer.GetNodesInTopologicalOrder(), graph_viewer); + result.push_back(ComputeCapability::Create(std::move(sub_graph))); + } else { auto partitions = GetSubGraphPartition(graph_viewer.GetNodesInTopologicalOrder(), unsupported_nodes); for (const auto& partition : partitions) { auto sub_graph = GetSubGraph(partition, graph_viewer); result.push_back(ComputeCapability::Create(std::move(sub_graph))); } - } else { - auto sub_graph = GetSubGraph(graph_viewer.GetNodesInTopologicalOrder(), graph_viewer); - result.push_back(ComputeCapability::Create(std::move(sub_graph))); } } else { InlinedVector candidates; diff --git a/onnxruntime/core/providers/cann/cann_execution_provider_info.cc b/onnxruntime/core/providers/cann/cann_execution_provider_info.cc index d6cf9fad70ae5..d1ba7544bc09e 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider_info.cc +++ b/onnxruntime/core/providers/cann/cann_execution_provider_info.cc @@ -20,7 +20,6 @@ constexpr const char* kDeviceId = "device_id"; constexpr const char* kMemLimit = "npu_mem_limit"; constexpr const char* kArenaExtendStrategy = "arena_extend_strategy"; constexpr const char* kEnableCannGraph = "enable_cann_graph"; -constexpr const char* kEnableCannSubGraph = "enable_cann_subgraph"; constexpr const char* kDumpGraphs = "dump_graphs"; constexpr const char* kDumpOmModel = "dump_om_model"; constexpr const char* kPrecisionMode = "precision_mode"; @@ -59,7 +58,6 @@ CANNExecutionProviderInfo CANNExecutionProviderInfo::FromProviderOptions(const P cann::provider_option_names::kArenaExtendStrategy, arena_extend_strategy_mapping, info.arena_extend_strategy) .AddAssignmentToReference(cann::provider_option_names::kEnableCannGraph, info.enable_cann_graph) - .AddAssignmentToReference(cann::provider_option_names::kEnableCannSubGraph, info.enable_cann_subgraph) .AddAssignmentToReference(cann::provider_option_names::kDumpGraphs, info.dump_graphs) .AddAssignmentToReference(cann::provider_option_names::kDumpOmModel, info.dump_om_model) .AddAssignmentToReference(cann::provider_option_names::kPrecisionMode, info.precision_mode) @@ -76,7 +74,6 @@ ProviderOptions CANNExecutionProviderInfo::ToProviderOptions(const CANNExecution {cann::provider_option_names::kArenaExtendStrategy, EnumToName(arena_extend_strategy_mapping, info.arena_extend_strategy)}, {cann::provider_option_names::kEnableCannGraph, MakeStringWithClassicLocale(info.enable_cann_graph)}, - {cann::provider_option_names::kEnableCannSubGraph, MakeStringWithClassicLocale(info.enable_cann_subgraph)}, {cann::provider_option_names::kDumpGraphs, MakeStringWithClassicLocale(info.dump_graphs)}, {cann::provider_option_names::kDumpOmModel, MakeStringWithClassicLocale(info.dump_om_model)}, {cann::provider_option_names::kPrecisionMode, MakeStringWithClassicLocale(info.precision_mode)}, @@ -92,7 +89,6 @@ ProviderOptions CANNExecutionProviderInfo::ToProviderOptions(const OrtCANNProvid {cann::provider_option_names::kArenaExtendStrategy, EnumToName(arena_extend_strategy_mapping, ArenaExtendStrategy(info.arena_extend_strategy))}, {cann::provider_option_names::kEnableCannGraph, MakeStringWithClassicLocale(info.enable_cann_graph)}, - {cann::provider_option_names::kEnableCannSubGraph, MakeStringWithClassicLocale(info.enable_cann_subgraph)}, {cann::provider_option_names::kDumpGraphs, MakeStringWithClassicLocale(info.dump_graphs)}, {cann::provider_option_names::kDumpOmModel, MakeStringWithClassicLocale(info.dump_om_model)}, {cann::provider_option_names::kPrecisionMode, MakeStringWithClassicLocale(info.precision_mode)}, diff --git a/onnxruntime/core/providers/cann/cann_execution_provider_info.h b/onnxruntime/core/providers/cann/cann_execution_provider_info.h index 9c1f9eb03b67e..7ac43e9a8ed6f 100644 --- a/onnxruntime/core/providers/cann/cann_execution_provider_info.h +++ b/onnxruntime/core/providers/cann/cann_execution_provider_info.h @@ -18,7 +18,6 @@ struct CANNExecutionProviderInfo { size_t npu_mem_limit{std::numeric_limits::max()}; ArenaExtendStrategy arena_extend_strategy{ArenaExtendStrategy::kNextPowerOfTwo}; bool enable_cann_graph{true}; - bool enable_cann_subgraph{false}; bool dump_graphs{false}; bool dump_om_model{true}; std::string precision_mode; diff --git a/onnxruntime/core/providers/cann/cann_provider_factory.cc b/onnxruntime/core/providers/cann/cann_provider_factory.cc index d3dc86f588f1d..4a130b9b0ca20 100644 --- a/onnxruntime/core/providers/cann/cann_provider_factory.cc +++ b/onnxruntime/core/providers/cann/cann_provider_factory.cc @@ -76,7 +76,6 @@ struct CANN_Provider : Provider { info.npu_mem_limit = params->npu_mem_limit; info.arena_extend_strategy = params->arena_extend_strategy; info.enable_cann_graph = params->enable_cann_graph != 0; - info.enable_cann_subgraph = params->enable_cann_subgraph != 0; info.dump_graphs = params->dump_graphs != 0; info.dump_om_model = params->dump_om_model != 0; info.precision_mode = params->precision_mode; @@ -95,7 +94,6 @@ struct CANN_Provider : Provider { cann_options.npu_mem_limit = internal_options.npu_mem_limit; cann_options.arena_extend_strategy = internal_options.arena_extend_strategy; cann_options.enable_cann_graph = internal_options.enable_cann_graph; - cann_options.enable_cann_subgraph = internal_options.enable_cann_subgraph; cann_options.dump_graphs = internal_options.dump_graphs; cann_options.dump_om_model = internal_options.dump_om_model; cann_options.precision_mode = internal_options.precision_mode; diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc index 93b673f2df5bd..b7997ce86737a 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc @@ -20,6 +20,7 @@ #include "onnx_ctx_model_helper.h" #include "core/providers/cuda/shared_inc/cuda_call.h" #include "core/providers/cuda/cuda_graph.h" +#include "core/providers/cuda/math/unary_elementwise_ops_impl.h" #include "core/session/allocator_adapters.h" #include "cuda_runtime_api.h" #include "core/common/parse_string.h" @@ -84,6 +85,40 @@ struct ShutdownProtobuf { namespace onnxruntime { +namespace cuda { +template <> +void Impl_Cast( + cudaStream_t stream, + const int64_t* input_data, int32_t* output_data, + size_t count) { + return g_host->cuda__Impl_Cast(static_cast(stream), input_data, output_data, count); +} + +template <> +void Impl_Cast( + cudaStream_t stream, + const int32_t* input_data, int64_t* output_data, + size_t count) { + return g_host->cuda__Impl_Cast(static_cast(stream), input_data, output_data, count); +} + +template <> +void Impl_Cast( + cudaStream_t stream, + const double* input_data, float* output_data, + size_t count) { + return g_host->cuda__Impl_Cast(static_cast(stream), input_data, output_data, count); +} + +template <> +void Impl_Cast( + cudaStream_t stream, + const float* input_data, double* output_data, + size_t count) { + return g_host->cuda__Impl_Cast(static_cast(stream), input_data, output_data, count); +} +} // namespace cuda + void* OutputAllocator::reallocateOutputAsync(char const* /*tensorName*/, void* /*currentMemory*/, uint64_t size, uint64_t /*alignment*/, cudaStream_t /*stream*/) noexcept { // Some memory allocators return nullptr when allocating zero bytes, but TensorRT requires a non-null ptr @@ -337,19 +372,51 @@ bool ApplyProfileShapesFromProviderOptions(std::vector(); \ + skip_input_binding_allowed = false; \ + if (input_tensor_ptr != nullptr && elem_cnt > 0) { \ + scratch_buffers.push_back(IAllocator::MakeUniquePtrFromOrtAllocator(alloc, elem_cnt * sizeof(DstT))); \ + data = scratch_buffers.back().get(); \ + cuda::Impl_Cast(stream, input_tensor_ptr, reinterpret_cast(data), elem_cnt); \ + } else { \ + scratch_buffers.push_back(IAllocator::MakeUniquePtrFromOrtAllocator(alloc, 1)); \ + data = scratch_buffers.back().get(); \ + } \ + break; \ + } + #define CASE_GET_OUTPUT_TENSOR(DATA_TYPE, SrcT) \ case DATA_TYPE: { \ auto output_tensor_ptr = output_tensor.GetTensorMutableData(); \ data_ptr = output_tensor_ptr; \ if (output_tensor_ptr != nullptr && elem_cnt > 0) { \ - buffer = output_tensor_ptr; \ + buffers[output_name] = output_tensor_ptr; \ } else { \ scratch_buffers.push_back(IAllocator::MakeUniquePtrFromOrtAllocator(alloc, 1)); \ - buffer = scratch_buffers.back().get(); \ + buffers[output_name] = scratch_buffers.back().get(); \ } \ break; \ } +#define CASE_GET_CAST_OUTPUT_TENSOR(DATA_TYPE, SrcT, DstT) \ + case DATA_TYPE: { \ + auto output_tensor_ptr = output_tensor.GetTensorMutableData(); \ + data_ptr = output_tensor_ptr; \ + skip_output_binding_allowed = false; \ + if (output_tensor_ptr != nullptr && elem_cnt > 0) { \ + scratch_buffers.push_back(IAllocator::MakeUniquePtrFromOrtAllocator(alloc, elem_cnt * sizeof(DstT))); \ + buffers[output_name] = scratch_buffers.back().get(); \ + output_dim_sizes[i] = static_cast(elem_cnt); \ + } else { \ + scratch_buffers.push_back(IAllocator::MakeUniquePtrFromOrtAllocator(alloc, 1)); \ + buffers[output_name] = scratch_buffers.back().get(); \ + output_dim_sizes[i] = 1; \ + } \ + break; \ + } + #define CASE_COPY_TENSOR(DATA_TYPE, DstT) \ case DATA_TYPE: { \ auto output_tensor_ptr = output_tensor.GetTensorMutableData(); \ @@ -359,6 +426,15 @@ bool ApplyProfileShapesFromProviderOptions(std::vector(); \ + if (output_tensor_ptr != nullptr && elem_cnt > 0) { \ + cuda::Impl_Cast(stream, reinterpret_cast(allocator->getBuffer()), reinterpret_cast(output_tensor_ptr), elem_cnt); \ + } \ + break; \ + } + /* * Set Nv executio context input. * @@ -481,6 +557,7 @@ Status BindContextInput(Ort::KernelContext& ctx, CASE_GET_INPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, uint8_t) CASE_GET_INPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, int32_t) CASE_GET_INPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, int64_t) + CASE_GET_CAST_INPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, double, float) default: { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "NvTensorRTRTX EP input onnx tensor data type: " + std::to_string(tensor_type) + " not supported."); @@ -505,6 +582,8 @@ Status BindContextInput(Ort::KernelContext& ctx, * param output_type - Data type of the output * param i - Output iteration index * param output_tensors - Output iteration index to output's ORT value + * param output_dim_sizes - Output iteration index to the multiplocation of its shape's dimensions + * param dds_output_set - DDS output set * param dds_output_allocator_map - DDS output to its allocator * param scratch_buffer - The allocation buffer created by TRT EP * param allocator - ORT allocator @@ -516,11 +595,16 @@ Status BindContextOutput(Ort::KernelContext& ctx, const char* output_name, size_t output_index, size_t output_type, + size_t i, + std::unordered_map& output_tensors, + std::unordered_map& output_dim_sizes, DDSOutputAllocatorMap& dds_output_allocator_map, std::vector>& scratch_buffers, OrtAllocator* alloc, + std::unordered_map& buffers, nvinfer1::Dims& dims, - void*& data_ptr) { + void*& data_ptr, + bool& skip_output_binding_allowed) { // Get output shape dims = trt_context->getTensorShape(output_name); int nb_dims = dims.nbDims; @@ -550,11 +634,10 @@ Status BindContextOutput(Ort::KernelContext& ctx, data_ptr = nullptr; // Set data_ptr to nullptr for DDS output binding. } } else { - auto output_tensor = ctx.GetOutput(output_index, dims.d, nb_dims); + output_tensors[i] = ctx.GetOutput(output_index, dims.d, nb_dims); + auto& output_tensor = output_tensors[i]; const auto elem_cnt = output_tensor.GetTensorTypeAndShapeInfo().GetElementCount(); - void* buffer = nullptr; - switch (output_type) { // below macros set data_ptr and skip_output_binding_allowed variables CASE_GET_OUTPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, float) @@ -565,12 +648,13 @@ Status BindContextOutput(Ort::KernelContext& ctx, CASE_GET_OUTPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, uint8_t) CASE_GET_OUTPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, int32_t) CASE_GET_OUTPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, int64_t) + CASE_GET_CAST_OUTPUT_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, double, float) default: { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "NvTensorRTRTX EP output tensor data type: " + std::to_string(output_type) + " not supported."); } } - trt_context->setTensorAddress(output_name, buffer); + trt_context->setTensorAddress(output_name, buffers[output_name]); } return Status::OK(); @@ -627,6 +711,7 @@ Status BindKernelOutput(Ort::KernelContext& ctx, CASE_COPY_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, uint8_t) CASE_COPY_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, int32_t) CASE_COPY_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, int64_t) + CASE_CAST_TENSOR(ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, float, double) default: { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "NvTensorRTRTX EP output tensor data type: " + std::to_string(output_type) + " not supported."); @@ -2752,6 +2837,7 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr } // Save TRT engine, other TRT objects and input/output info to map + parsers_.emplace(fused_node.Name(), std::move(trt_parser)); engines_.emplace(fused_node.Name(), std::move(trt_engine)); contexts_.emplace(fused_node.Name(), std::move(trt_context)); networks_.emplace(fused_node.Name(), std::move(trt_network)); @@ -2767,7 +2853,7 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr compute_info.create_state_func = [=](ComputeContext* context, FunctionState* state) { std::unique_ptr p = std::make_unique(); *p = {context->allocate_func, context->release_func, context->allocator_handle, context->node_name, builder_.get(), - &engines_[context->node_name], &contexts_[context->node_name], + &parsers_[context->node_name], &engines_[context->node_name], &contexts_[context->node_name], &networks_[context->node_name], input_info_[context->node_name], output_info_[context->node_name], input_shape_ranges_[context->node_name], &tensorrt_mu_, engine_cache_enable_, cache_path_, @@ -2805,6 +2891,7 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr auto trt_engine = trt_state->engine->get(); auto trt_context = trt_state->context->get(); auto trt_profiles = trt_state->profiles; + int num_outputs = static_cast(output_indexes.size()); std::unordered_set input_names; if (alloc_ == nullptr) { @@ -2879,7 +2966,16 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr /* * Set output shapes and bind output buffers */ + std::unordered_map buffers; + buffers.reserve(num_outputs); + using OutputOrtValue = Ort::UnownedValue; + std::unordered_map output_tensors; + output_tensors.reserve(num_outputs); + std::unordered_map output_dim_sizes; + output_dim_sizes.reserve(num_outputs); + if (require_io_binding) { + bool skip_output_binding_allowed = true; for (size_t i = 0, end = output_binding_names.size(); i < end; ++i) { char const* output_name = output_binding_names[i]; @@ -2897,15 +2993,16 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr nvinfer1::Dims dims; void* data_ptr = nullptr; - - Status status = BindContextOutput(ctx, trt_context, output_name, output_index, output_type, - dds_output_allocator_map, scratch_buffers, alloc, dims, data_ptr); + Status status = BindContextOutput(ctx, trt_context, output_name, output_index, output_type, i, output_tensors, output_dim_sizes, + dds_output_allocator_map, scratch_buffers, alloc, buffers, dims, data_ptr, skip_output_binding_allowed); if (status != Status::OK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, status.ErrorMessage()); } trt_state->output_tensors[output_index] = TensorParams{data_ptr, dims}; } + + trt_state->skip_io_binding_allowed = trt_state->skip_io_binding_allowed | skip_output_binding_allowed; } // Set execution context memory @@ -2985,6 +3082,14 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr if (status != Status::OK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, status.ErrorMessage()); } + } else { + auto& output_tensor = output_tensors[i]; + if (output_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) { + auto output_tensor_ptr = output_tensor.GetTensorMutableData(); + if (output_tensor_ptr != nullptr) { + cuda::Impl_Cast(stream, reinterpret_cast(buffers[output_name]), output_tensor_ptr, output_dim_sizes[i]); + } + } } } @@ -3108,6 +3213,7 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(const Gra auto& dds_output_allocator_map = this->dds_output_allocator_maps_[fused_node_name]; auto trt_engine = trt_state->engine->get(); auto trt_context = trt_state->context->get(); + int num_outputs = static_cast(output_indexes.size()); std::unordered_map> shape_tensor_values; // This map holds "shape tensor -> shape values" for the shape tensor input across this inference run std::unordered_map> shape_tensor_values_int64; // same as above but for int64 shape tensor input @@ -3177,7 +3283,16 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(const Gra /* * Set output shapes and bind output buffers */ + std::unordered_map buffers; + buffers.reserve(num_outputs); + using OutputOrtValue = Ort::UnownedValue; + std::unordered_map output_tensors; + output_tensors.reserve(num_outputs); + std::unordered_map output_dim_sizes; + output_dim_sizes.reserve(num_outputs); + if (require_io_binding) { + bool skip_output_binding_allowed = true; for (size_t i = 0, end = output_binding_names.size(); i < end; ++i) { char const* output_name = output_binding_names[i]; @@ -3196,14 +3311,16 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(const Gra nvinfer1::Dims dims; void* data_ptr = nullptr; - Status status = BindContextOutput(ctx, trt_context, output_name, output_index, output_type, - dds_output_allocator_map, scratch_buffers, alloc, dims, data_ptr); + Status status = BindContextOutput(ctx, trt_context, output_name, output_index, output_type, i, output_tensors, output_dim_sizes, + dds_output_allocator_map, scratch_buffers, alloc, buffers, dims, data_ptr, skip_output_binding_allowed); if (status != Status::OK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, status.ErrorMessage()); } trt_state->output_tensors[output_index] = TensorParams{data_ptr, dims}; } + + trt_state->skip_io_binding_allowed = trt_state->skip_io_binding_allowed | skip_output_binding_allowed; } // Set execution context memory @@ -3284,6 +3401,14 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(const Gra if (status != Status::OK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, status.ErrorMessage()); } + } else { + auto& output_tensor = output_tensors[i]; + if (output_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) { + auto output_tensor_ptr = output_tensor.GetTensorMutableData(); + if (output_tensor_ptr != nullptr) { + cuda::Impl_Cast(stream, reinterpret_cast(buffers[output_name]), output_tensor_ptr, output_dim_sizes[i]); + } + } } } diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h index 9e5fd03756f02..22b8314649757 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h @@ -195,6 +195,7 @@ struct TensorrtFuncState { AllocatorHandle allocator = nullptr; std::string fused_node_name; nvinfer1::IBuilder* builder; + tensorrt_ptr::unique_pointer* parser = nullptr; std::unique_ptr* engine = nullptr; std::unique_ptr* context = nullptr; std::unique_ptr* network = nullptr; @@ -385,6 +386,7 @@ class NvExecutionProvider : public IExecutionProvider { // In general, TensorRT objects are not thread safe; accesses to an object from different threads must be serialized by the client. // But there are still some thread safe operations, please see here https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#threading // For those non thread safe operations, TRT EP uses (1) lock_guard or (2) PerThreadContext to make sure synchronization. + std::unordered_map> parsers_; std::unordered_map> engines_; std::unordered_map> contexts_; std::unordered_map> builders_; diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc index a994c936970f6..541ca5ca7ab14 100644 --- a/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc +++ b/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc @@ -245,12 +245,6 @@ Status ConvOpBuilder::ProcessConv2D3DInputs(QnnModelWrapper& qnn_model_wrapper, bool is_graph_input = qnn_model_wrapper.IsGraphInput(input1_name); LOGS(logger, VERBOSE) << "Add HWCN Transpose node after input: " << input1_name; - if (!qnn_model_wrapper.IsQnnTensorWrapperExist(input1_name)) { - QnnTensorWrapper weight_tensor_wrapper; - ORT_RETURN_IF_ERROR(qnn_model_wrapper.MakeTensorWrapper(inputs[1], weight_tensor_wrapper)); - ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(weight_tensor_wrapper)), "Failed to add weight tensor."); - } - if (conv_type == OnnxConvType::kConv) { ORT_RETURN_IF_ERROR(qnn_model_wrapper.AddNchwToHwcnTranspose(node_unit.Index(), input1_name, @@ -431,7 +425,7 @@ Status ConvOpBuilder::ProcessConv1DInputs(QnnModelWrapper& qnn_model_wrapper, // // Input 1: weight - // We need to first reshape the weight in order to handle 1D convolutions with the Conv2d operator. + // We need to first reshape the weight inorder to handle 1D convolutions with the Conv2d operator. // Next, we have to transpose the weight because ORT layout transformations do not change the weight layout. // { @@ -517,12 +511,6 @@ Status ConvOpBuilder::ProcessConv1DInputs(QnnModelWrapper& qnn_model_wrapper, ORT_RETURN_IF(input_info.quant_param.IsPerChannel(), "Non-constant Conv inputs only support per-tensor quantization"); - if (!qnn_model_wrapper.IsQnnTensorWrapperExist(input1_name)) { - QnnTensorWrapper weight_tensor_wrapper; - ORT_RETURN_IF_ERROR(qnn_model_wrapper.MakeTensorWrapper(inputs[1], weight_tensor_wrapper)); - ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(weight_tensor_wrapper)), "Failed to add weight tensor."); - } - bool is_graph_input = qnn_model_wrapper.IsGraphInput(input1_name); LOGS(logger, VERBOSE) << "Adding Reshape (to 2D) and HWCN Transpose node after input: " << input1_name; ORT_RETURN_IF_ERROR(qnn_model_wrapper.AddReshapeNode(input1_name, diff --git a/onnxruntime/core/providers/shared_library/provider_ort_api_init.cc b/onnxruntime/core/providers/shared_library/provider_ort_api_init.cc index f8d88b07f6dd5..9fa2551e53c23 100644 --- a/onnxruntime/core/providers/shared_library/provider_ort_api_init.cc +++ b/onnxruntime/core/providers/shared_library/provider_ort_api_init.cc @@ -24,7 +24,7 @@ std::once_flag init; } // namespace void InitProviderOrtApi() { - std::call_once(init, []() { Ort::InitApi(Provider_GetHost()->OrtGetApiBase()->GetApi(ORT_API_VERSION)); }); + std::call_once(init, []() { Ort::Global::api_ = Provider_GetHost()->OrtGetApiBase()->GetApi(ORT_API_VERSION); }); } -} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index 580fbfbdba0b0..5fc0b8900730b 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -229,7 +229,7 @@ int vitisai_ep_set_ep_dynamic_options( struct MyCustomOpKernel : OpKernel { MyCustomOpKernel(const OpKernelInfo& info, const OrtCustomOp& op) : OpKernel(info), op_(op) { op_kernel_ = - op_.CreateKernel(&op_, &Ort::GetApi(), reinterpret_cast(&info)); + op_.CreateKernel(&op_, Ort::Global::api_, reinterpret_cast(&info)); } ~MyCustomOpKernel() override { op_.KernelDestroy(op_kernel_); } @@ -332,8 +332,8 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { InitProviderOrtApi(); set_version_info(the_global_api); the_global_api.host_ = Provider_GetHost(); - assert(&Ort::GetApi() != nullptr); - the_global_api.ort_api_ = &Ort::GetApi(); + assert(Ort::Global::api_ != nullptr); + the_global_api.ort_api_ = Ort::Global::api_; the_global_api.model_load = [](const std::string& filename) -> Model* { auto model_proto = ONNX_NAMESPACE::ModelProto::Create(); auto& logger = logging::LoggingManager::DefaultLogger(); diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index f3e2a8ce7ba7b..ad0a1ad137f06 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -3036,7 +3036,7 @@ ORT_API_STATUS_IMPL(OrtApis::Node_GetAttributeByName, _In_ const OrtNode* node, API_IMPL_END } -ORT_API_STATUS_IMPL(OrtApis::OpAttr_GetTensorAttributeAsOrtValue, _In_ const OrtOpAttr* attribute, _Outptr_result_maybenull_ OrtValue** attr_tensor) { +ORT_API_STATUS_IMPL(OrtApis::Node_GetTensorAttributeAsOrtValue, _In_ const OrtNode* node, _In_ const OrtOpAttr* attribute, _Outptr_result_maybenull_ OrtValue** attr_tensor) { API_IMPL_BEGIN if (attr_tensor == nullptr) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "attr_tensor argument is null"); @@ -3045,39 +3045,7 @@ ORT_API_STATUS_IMPL(OrtApis::OpAttr_GetTensorAttributeAsOrtValue, _In_ const Ort return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "attribute argument is null"); } - const auto* attr_proto = reinterpret_cast(attribute); - - if (attr_proto->type() != onnx::AttributeProto::TENSOR) { - return OrtApis::CreateStatus(OrtErrorCode::ORT_INVALID_ARGUMENT, "This OrtOpAttr instance is not a 'TENSOR' attribute"); - } - - const auto& tensor_proto = attr_proto->t(); - - // Check that TensorProto is valid. - if (!utils::HasDataType(tensor_proto)) { - return OrtApis::CreateStatus(OrtErrorCode::ORT_INVALID_ARGUMENT, "Tensor proto doesn't have data type."); - } - - if (!ONNX_NAMESPACE::TensorProto::DataType_IsValid(tensor_proto.data_type())) { - return OrtApis::CreateStatus(OrtErrorCode::ORT_INVALID_ARGUMENT, "Tensor proto has invalid data type."); - } - - if (utils::HasExternalData(tensor_proto)) { - return OrtApis::CreateStatus(OrtErrorCode::ORT_INVALID_ARGUMENT, - "Tensor proto with external data for value attribute is not supported."); - } - - // Initialize OrtValue for tensor attribute. - auto tensor_attribute_value = std::make_unique(); - AllocatorPtr tensor_attribute_allocator = CPUAllocator::DefaultInstance(); - // The tensor in the 'Tensor' attribute's TensorProto is stored inline, not in an external file. - // Therefore, the 'model_path' passed to TensorProtoToOrtValue() may be an empty path. - std::filesystem::path model_path; - ORT_API_RETURN_IF_STATUS_NOT_OK(utils::TensorProtoToOrtValue(Env::Default(), model_path, tensor_proto, - tensor_attribute_allocator, *tensor_attribute_value)); - - *attr_tensor = tensor_attribute_value.release(); - + ORT_API_RETURN_IF_STATUS_NOT_OK(node->GetTensorAttributeAsOrtValue(attribute, *attr_tensor)); return nullptr; API_IMPL_END } @@ -4166,7 +4134,7 @@ static constexpr OrtApi ort_api_1_to_23 = { &OrtApis::Node_GetNumAttributes, &OrtApis::Node_GetAttributes, &OrtApis::Node_GetAttributeByName, - &OrtApis::OpAttr_GetTensorAttributeAsOrtValue, + &OrtApis::Node_GetTensorAttributeAsOrtValue, &OrtApis::OpAttr_GetType, &OrtApis::OpAttr_GetName, &OrtApis::Node_GetNumSubgraphs, diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index 6dc4cf9d195cc..e62149d04a16c 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -687,7 +687,7 @@ ORT_API_STATUS_IMPL(Node_GetAttributes, _In_ const OrtNode* node, _Out_writes_(num_attributes) const OrtOpAttr** attributes, _In_ size_t num_attributes); ORT_API_STATUS_IMPL(Node_GetAttributeByName, _In_ const OrtNode* node, _In_ const char* attribute_name, _Outptr_result_maybenull_ const OrtOpAttr** attribute); -ORT_API_STATUS_IMPL(OpAttr_GetTensorAttributeAsOrtValue, _In_ const OrtOpAttr* attribute, +ORT_API_STATUS_IMPL(Node_GetTensorAttributeAsOrtValue, _In_ const OrtNode* node, _In_ const OrtOpAttr* attribute, _Outptr_result_maybenull_ OrtValue** attr_tensor); ORT_API_STATUS_IMPL(OpAttr_GetType, _In_ const OrtOpAttr* attribute, _Out_ OrtOpAttrType* type); ORT_API_STATUS_IMPL(OpAttr_GetName, _In_ const OrtOpAttr* attribute, _Outptr_ const char** name); diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.cc b/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.cc index 42b65239de92c..d6e51a44c1c69 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.cc +++ b/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.cc @@ -4,8 +4,6 @@ #include "core/session/plugin_ep/ep_factory_provider_bridge.h" #include "core/providers/shared_library/provider_host_api.h" -#include "core/session/plugin_ep/ep_library_plugin.h" -#include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" namespace onnxruntime { OrtStatus* ProviderBridgeEpFactory::GetSupportedDevices(EpFactoryInternal& ep_factory, @@ -22,11 +20,6 @@ OrtStatus* ProviderBridgeEpFactory::GetSupportedDevices(EpFactoryInternal& ep_fa auto* ep_device = ep_devices[i]; if (ep_device) { ep_device->ep_factory = &ep_factory; - - // Add library path to EP metadata if available - if (library_path_.has_value()) { - ep_device->ep_metadata.Add(kOrtEpDevice_EpMetadataKey_LibraryPath, library_path_->string()); - } } } diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.h b/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.h index 8c5ef526baba1..437af62dc2c0c 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_provider_bridge.h @@ -3,10 +3,6 @@ #pragma once -#include -#include -#include - #include "core/framework/error_code_helper.h" #include "core/session/abi_devices.h" #include "core/session/abi_session_options_impl.h" @@ -16,14 +12,12 @@ namespace onnxruntime { class ProviderBridgeEpFactory : public EpFactoryInternalImpl { public: - ProviderBridgeEpFactory(OrtEpFactory& ep_factory, ProviderLibrary& provider_library, - std::optional library_path = std::nullopt) + ProviderBridgeEpFactory(OrtEpFactory& ep_factory, ProviderLibrary& provider_library) : EpFactoryInternalImpl(ep_factory.GetName(&ep_factory), ep_factory.GetVendor(&ep_factory), ep_factory.GetVendorId(&ep_factory)), ep_factory_{ep_factory}, - provider_library_{provider_library}, - library_path_{std::move(library_path)} { + provider_library_{provider_library} { } private: @@ -65,9 +59,8 @@ class ProviderBridgeEpFactory : public EpFactoryInternalImpl { return ep_factory_.CreateSyncStreamForDevice(&ep_factory_, device, stream_options, stream); } - OrtEpFactory& ep_factory_; - ProviderLibrary& provider_library_; - std::optional library_path_; + OrtEpFactory& ep_factory_; // OrtEpFactory from the provider bridge EP + ProviderLibrary& provider_library_; // ProviderLibrary from the provider bridge EP }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library.h b/onnxruntime/core/session/plugin_ep/ep_library.h index af5bc23143e33..24ab74e1c77fc 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library.h +++ b/onnxruntime/core/session/plugin_ep/ep_library.h @@ -23,7 +23,6 @@ class EpLibrary { virtual Status Load() { return Status::OK(); } virtual const std::vector& GetFactories() = 0; // valid after Load() virtual Status Unload() { return Status::OK(); } - virtual ~EpLibrary() = default; ORT_DISALLOW_COPY_AND_ASSIGNMENT(EpLibrary); diff --git a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc index da94a9f12ba9d..06cf54aea4071 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc +++ b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc @@ -4,7 +4,6 @@ #include "core/session/plugin_ep/ep_library_provider_bridge.h" #include "core/session/plugin_ep/ep_factory_provider_bridge.h" -#include "core/session/plugin_ep/ep_library_plugin.h" namespace onnxruntime { Status EpLibraryProviderBridge::Load() { @@ -27,9 +26,8 @@ Status EpLibraryProviderBridge::Load() { // to do this we need to capture `factory` and plug it in to is_supported_fn and create_fn. // we also need to update any returned OrtEpDevice instances to swap the wrapper EpFactoryInternal in so that we can // call Provider::CreateIExecutionProvider in EpFactoryInternal::CreateIExecutionProvider. - for (const auto& factory : ep_library_plugin_->GetFactories()) { - auto factory_impl = std::make_unique(*factory, *provider_library_, library_path_); + auto factory_impl = std::make_unique(*factory, *provider_library_); auto internal_factory = std::make_unique(std::move(factory_impl)); factory_ptrs_.push_back(internal_factory.get()); diff --git a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h index 45277b2828f56..c7e8ebefc3785 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h +++ b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h @@ -21,11 +21,9 @@ namespace onnxruntime { class EpLibraryProviderBridge : public EpLibrary { public: EpLibraryProviderBridge(std::unique_ptr provider_library, - std::unique_ptr ep_library_plugin, - std::optional library_path = std::nullopt) + std::unique_ptr ep_library_plugin) : provider_library_{std::move(provider_library)}, - ep_library_plugin_{std::move(ep_library_plugin)}, - library_path_{std::move(library_path)} { + ep_library_plugin_{std::move(ep_library_plugin)} { } const char* RegistrationName() const override { @@ -55,9 +53,6 @@ class EpLibraryProviderBridge : public EpLibrary { // implement EpFactoryInternal::CreateIExecutionProvider by calling Provider::CreateIExecutionProvider. std::unique_ptr ep_library_plugin_; - // Library path for EP metadata - std::optional library_path_; - std::vector> factories_; std::vector factory_ptrs_; // for convenience std::vector internal_factory_ptrs_; // for convenience diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index f82cbcf63ca62..41cf8be1d1412 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -2902,7 +2902,6 @@ ORT_API_STATUS_IMPL(OrtApis::CreateCANNProviderOptions, _Outptr_ OrtCANNProvider options->npu_mem_limit = SIZE_MAX; options->arena_extend_strategy = static_cast(0); options->enable_cann_graph = 1; - options->enable_cann_subgraph = 0; options->dump_graphs = 0; options->dump_om_model = 1; options->default_memory_arena_cfg = nullptr; diff --git a/onnxruntime/core/session/utils.cc b/onnxruntime/core/session/utils.cc index 7da7fabb15b15..d4041dfce5a7a 100644 --- a/onnxruntime/core/session/utils.cc +++ b/onnxruntime/core/session/utils.cc @@ -421,14 +421,13 @@ Status LoadPluginOrProviderBridge(const std::string& registration_name, << (is_provider_bridge ? " as a provider bridge" : " as a plugin"); // create EpLibraryPlugin to ensure CreateEpFactories and ReleaseEpFactory are available - auto ep_library_plugin = std::make_unique(registration_name, resolved_library_path); + auto ep_library_plugin = std::make_unique(registration_name, std::move(resolved_library_path)); ORT_RETURN_IF_ERROR(ep_library_plugin->Load()); if (is_provider_bridge) { // wrap the EpLibraryPlugin with EpLibraryProviderBridge to add to directly create an IExecutionProvider auto ep_library_provider_bridge = std::make_unique(std::move(provider_library), - std::move(ep_library_plugin), - resolved_library_path); + std::move(ep_library_plugin)); ORT_RETURN_IF_ERROR(ep_library_provider_bridge->Load()); internal_factories = ep_library_provider_bridge->GetInternalFactories(); ep_library = std::move(ep_library_provider_bridge); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index eb06a65ad5330..24554560b4dde 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1575,17 +1575,6 @@ void addGlobalMethods(py::module& m) { R"pbdoc(Get the list of available OrtEpDevice instances.)pbdoc", py::return_value_policy::reference); - m.def( - "get_model_compatibility_for_ep_devices", - [](const std::vector& ep_devices, - const std::string& compatibility_info) -> OrtCompiledModelCompatibility { - OrtCompiledModelCompatibility status = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; - Ort::ThrowOnError(Ort::GetApi().GetModelCompatibilityForEpDevices( - ep_devices.data(), ep_devices.size(), compatibility_info.c_str(), &status)); - return status; - }, - R"pbdoc("Validate a compiled model's compatibility information for one or more EP devices.)pbdoc"); - #if defined(USE_OPENVINO) || defined(USE_OPENVINO_PROVIDER_INTERFACE) m.def( "get_available_openvino_device_ids", []() -> std::vector { @@ -1770,12 +1759,6 @@ void addObjectMethods(py::module& m, ExecutionProviderRegistrationFn ep_registra .value("PRIORITY_BASED", ExecutionOrder::PRIORITY_BASED) .value("MEMORY_EFFICIENT", ExecutionOrder::MEMORY_EFFICIENT); - py::enum_(m, "OrtCompiledModelCompatibility") - .value("EP_NOT_APPLICABLE", OrtCompiledModelCompatibility_EP_NOT_APPLICABLE) - .value("EP_SUPPORTED_OPTIMAL", OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL) - .value("EP_SUPPORTED_PREFER_RECOMPILATION", OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION) - .value("EP_UNSUPPORTED", OrtCompiledModelCompatibility_EP_UNSUPPORTED); - py::enum_(m, "OrtAllocatorType") .value("INVALID", OrtInvalidAllocator) .value("ORT_DEVICE_ALLOCATOR", OrtDeviceAllocator) @@ -1799,7 +1782,7 @@ void addObjectMethods(py::module& m, ExecutionProviderRegistrationFn ep_registra type = OrtDevice::GPU; vendor = OrtDevice::VendorIds::MICROSOFT; } else if (type == OrtDevice::GPU) { -#if USE_CUDA || USE_NV || USE_NV_PROVIDER_INTERFACE || USE_CUDA_PROVIDER_INTERFACE +#if USE_CUDA vendor = OrtDevice::VendorIds::NVIDIA; #elif USE_ROCM || USE_MIGRAPHX vendor = OrtDevice::VendorIds::AMD; diff --git a/onnxruntime/python/tools/quantization/execution_providers/qnn/preprocess.py b/onnxruntime/python/tools/quantization/execution_providers/qnn/preprocess.py index a12aca47f5b65..191edc4c6390d 100644 --- a/onnxruntime/python/tools/quantization/execution_providers/qnn/preprocess.py +++ b/onnxruntime/python/tools/quantization/execution_providers/qnn/preprocess.py @@ -6,15 +6,15 @@ from __future__ import annotations import logging -import tempfile from pathlib import Path import onnx -from ....tools.onnx_model_utils import fix_output_shapes, make_input_shape_fixed, optimize_model +from ....tools.onnx_model_utils import fix_output_shapes, make_input_shape_fixed from ....tools.remove_initializer_from_input import remove_initializer_from_input from ...fusions import FusionGelu, FusionLayerNormalization from ...onnx_model import ONNXModel +from ...quant_utils import save_and_reload_model_with_shape_infer from .fusion_lpnorm import FusionLpNormalization from .fusion_spacetodepth import FusionSpaceToDepth @@ -93,7 +93,7 @@ def qnn_preprocess_model( """ modified = False model = model_input if isinstance(model_input, onnx.ModelProto) else onnx.load_model(model_input) - model = save_and_reload_optimize_model(model, shape_infer=True) + model = save_and_reload_model_with_shape_infer(model) onnx_model = ONNXModel(model) # Optionally, fix the dynamic input shapes. @@ -178,24 +178,6 @@ def qnn_preprocess_model( return modified -def save_and_reload_optimize_model(model: onnx.ModelProto, shape_infer: bool) -> onnx.ModelProto: - with tempfile.TemporaryDirectory(prefix="ort.qnn_preproc.") as qnn_preproc_tmp_dir: - model_in_path = Path(qnn_preproc_tmp_dir).joinpath("qnn_proc_input.onnx") - onnx.save_model(model, model_in_path, save_as_external_data=True) - if shape_infer: - model_infer_path = Path(qnn_preproc_tmp_dir).joinpath("qnn_proc_infer.onnx") - onnx.shape_inference.infer_shapes_path(str(model_in_path), str(model_infer_path)) - model_in_path = model_infer_path - model_out_path = Path(qnn_preproc_tmp_dir).joinpath("qnn_proc_output.onnx") - optimize_model(model_in_path, model_out_path) - ret_model = onnx.load_model(model_out_path) - ret_metaprops = {"onnx.infer": "onnxruntime.tools.qnn.preprocess"} - if ret_model.metadata_props: - ret_metaprops.update(ret_model.metadata_props) - onnx.helper.set_model_props(ret_model, ret_metaprops) - return ret_model - - class InputOutputNameMap: def __init__( self, diff --git a/onnxruntime/test/autoep/library/ep_arena.h b/onnxruntime/test/autoep/library/ep_arena.h index caa2c61db835f..641f3ce3f7b17 100644 --- a/onnxruntime/test/autoep/library/ep_arena.h +++ b/onnxruntime/test/autoep/library/ep_arena.h @@ -21,10 +21,7 @@ limitations under the License. #include #include -#define ORT_API_MANUAL_INIT #include "onnxruntime_cxx_api.h" -#undef ORT_API_MANUAL_INIT - #include "ep_allocator.h" #include "example_plugin_ep_utils.h" diff --git a/onnxruntime/test/contrib_ops/moe_test.cc b/onnxruntime/test/contrib_ops/moe_test.cc index 0690b8894eb7a..ed7ca998e0b86 100644 --- a/onnxruntime/test/contrib_ops/moe_test.cc +++ b/onnxruntime/test/contrib_ops/moe_test.cc @@ -144,12 +144,6 @@ static void RunQMoETest(const std::vector& input, const std::vector("k", static_cast(top_k)); cpu_tester.AddAttribute("activation_type", activation_type); @@ -1329,13 +1323,6 @@ TEST(MoETest, QMoETest_Mixtral_Int4) { // CPU-specific QMoE tests TEST(MoETest, QMoETest_CPU_Int4_MLAS) { -#ifdef USE_MLAS - // Skip this test if we're not testing CPU execution provider - auto cpu_ep = DefaultCpuExecutionProvider(); - if (!cpu_ep) { - GTEST_SKIP() << "CPU execution provider not available"; - } - int num_rows = 2; int num_experts = 2; int hidden_size = 32; @@ -1400,19 +1387,9 @@ TEST(MoETest, QMoETest_CPU_Int4_MLAS) { std::vector> cpu_execution_providers; cpu_execution_providers.push_back(DefaultCpuExecutionProvider()); cpu_tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &cpu_execution_providers); -#else - GTEST_SKIP() << "Skipping CPU QMoE test"; -#endif } TEST(MoETest, QMoETest_CPU_Int8_MLAS) { -#ifdef USE_MLAS - // Skip this test if we're not testing CPU execution provider - auto cpu_ep = DefaultCpuExecutionProvider(); - if (!cpu_ep) { - GTEST_SKIP() << "CPU execution provider not available"; - } - // Test CPU implementation with 8-bit quantization - CPU ONLY int num_rows = 1; int num_experts = 2; @@ -1469,19 +1446,9 @@ TEST(MoETest, QMoETest_CPU_Int8_MLAS) { std::vector> cpu_execution_providers; cpu_execution_providers.push_back(DefaultCpuExecutionProvider()); cpu_tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &cpu_execution_providers); -#else - GTEST_SKIP() << "Skipping CPU QMoE test"; -#endif } TEST(MoETest, QMoETest_CPU_FC3_Error) { -#ifdef USE_MLAS - // Skip this test if we're not testing CPU execution provider - auto cpu_ep = DefaultCpuExecutionProvider(); - if (!cpu_ep) { - GTEST_SKIP() << "CPU execution provider not available"; - } - // Test that CPU throws error when FC3 gating is provided - CPU ONLY int num_rows = 1; int num_experts = 2; @@ -1539,19 +1506,9 @@ TEST(MoETest, QMoETest_CPU_FC3_Error) { // Expect this to fail with FC3 not implemented error cpu_tester.Run(OpTester::ExpectResult::kExpectFailure, "FC3 gating is not yet implemented", {}, nullptr, &cpu_execution_providers); -#else - GTEST_SKIP() << "Skipping CPU QMoE test"; -#endif } TEST(MoETest, QMoETest_CPU_SwiGLU_Int4) { -#ifdef USE_MLAS - // Skip this test if we're not testing CPU execution provider - auto cpu_ep = DefaultCpuExecutionProvider(); - if (!cpu_ep) { - GTEST_SKIP() << "CPU execution provider not available"; - } - // Test CPU implementation with 4-bit quantization and SwiGLU activation int num_rows = 2; int num_experts = 2; @@ -1616,18 +1573,9 @@ TEST(MoETest, QMoETest_CPU_SwiGLU_Int4) { std::vector> cpu_execution_providers; cpu_execution_providers.push_back(DefaultCpuExecutionProvider()); cpu_tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &cpu_execution_providers); -#else - GTEST_SKIP() << "Skipping CPU QMoE test"; -#endif } TEST(MoETest, QMoETest_CPU_SwiGLU_Int8) { -#ifdef USE_MLAS - // Skip this test if we're not testing CPU execution provider - auto cpu_ep = DefaultCpuExecutionProvider(); - if (!cpu_ep) { - GTEST_SKIP() << "CPU execution provider not available"; - } // Test CPU implementation with 8-bit quantization and SwiGLU activation int num_rows = 1; int num_experts = 2; @@ -1685,9 +1633,6 @@ TEST(MoETest, QMoETest_CPU_SwiGLU_Int8) { std::vector> cpu_execution_providers; cpu_execution_providers.push_back(DefaultCpuExecutionProvider()); cpu_tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &cpu_execution_providers); -#else - GTEST_SKIP() << "Skipping CPU QMoE test"; -#endif } #endif diff --git a/onnxruntime/test/framework/ep_compatibility_test.cc b/onnxruntime/test/framework/ep_compatibility_test.cc index a8a83fbe5ceb6..ee82d4683ab73 100644 --- a/onnxruntime/test/framework/ep_compatibility_test.cc +++ b/onnxruntime/test/framework/ep_compatibility_test.cc @@ -15,7 +15,6 @@ #include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" #include "core/session/utils.h" #include "core/session/onnxruntime_c_api.h" -#include "core/session/onnxruntime_cxx_api.h" #include "core/session/abi_session_options_impl.h" #include "core/framework/error_code_helper.h" #include "dummy_provider.h" @@ -500,31 +499,3 @@ TEST(EpCompatibilityCapiTest, CpuEpReturnsNotApplicableIfNoValidation) { api->ReleaseEnv(env); } - -// ----------------------------- -// C++ API unit tests -// ----------------------------- - -TEST(EpCompatibilityCxxApiTest, SingleDeviceCpuProvider) { - Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatCxx"}; - auto devices = env.GetEpDevices(); - ASSERT_FALSE(devices.empty()); - - std::vector selected; - for (const auto& d : devices) { - if (std::string{d.EpName()} == "CPUExecutionProvider") { - selected.push_back(d); - break; - } - } - - ASSERT_FALSE(selected.empty()); - - // Pick a status that the CPU EP would never return to ensure the value is set correctly. - OrtCompiledModelCompatibility status = OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION; - ASSERT_NO_FATAL_FAILURE({ - status = Ort::GetModelCompatibilityForEpDevices(selected, "arbitrary-compat-string"); - }); - - ASSERT_TRUE(status == OrtCompiledModelCompatibility_EP_NOT_APPLICABLE); -} \ No newline at end of file diff --git a/onnxruntime/test/platform/device_discovery_test.cc b/onnxruntime/test/platform/device_discovery_test.cc index 6b43ccbc8f670..21ddf9a5b1cd7 100644 --- a/onnxruntime/test/platform/device_discovery_test.cc +++ b/onnxruntime/test/platform/device_discovery_test.cc @@ -25,9 +25,9 @@ TEST(DeviceDiscoveryTest, HasCpuDevice) { const auto cpu_devices = GetDevicesByType(OrtHardwareDeviceType_CPU); ASSERT_GT(cpu_devices.size(), 0); -#if defined(CPUINFO_SUPPORTED) +#if !defined(__wasm__) ASSERT_NE(cpu_devices[0].vendor_id, 0); -#endif // defined(CPUINFO_SUPPORTED) +#endif // !defined(__WASM__) } } // namespace onnxruntime::test diff --git a/onnxruntime/test/python/onnxruntime_test_python_ep_compatibility.py b/onnxruntime/test/python/onnxruntime_test_python_ep_compatibility.py deleted file mode 100644 index 8e69fdf088103..0000000000000 --- a/onnxruntime/test/python/onnxruntime_test_python_ep_compatibility.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import os -import platform -import sys -import unittest - -from onnxruntime.capi.onnxruntime_pybind11_state import ( - OrtCompiledModelCompatibility, - get_ep_devices, - get_model_compatibility_for_ep_devices, -) - -# handle change from python 3.8 and on where loading a dll from the current directory needs to be explicitly allowed. -if platform.system() == "Windows" and sys.version_info.major >= 3 and sys.version_info.minor >= 8: # noqa: YTT204 - os.add_dll_directory(os.getcwd()) - - -class TestEpCompatibility(unittest.TestCase): - def test_invalid_args(self): - # empty devices - with self.assertRaises(RuntimeError): - get_model_compatibility_for_ep_devices([], "info") - # None compatibility info should raise TypeError before native call - with self.assertRaises(TypeError): - get_model_compatibility_for_ep_devices(get_ep_devices(), None) # type: ignore[arg-type] - - def test_basic_smoke(self): - devices = list(get_ep_devices()) - if not devices: - self.skipTest("No EP devices available in this build") - - # Always select CPUExecutionProvider; skip if not present. - cpu_devices = [d for d in devices if getattr(d, "ep_name", None) == "CPUExecutionProvider"] - if not cpu_devices: - self.skipTest("CPUExecutionProvider not available in this build") - selected = [cpu_devices[0]] - - # API requires all devices belong to the same EP; we pass only one. - status = get_model_compatibility_for_ep_devices(selected, "arbitrary-compat-string") - self.assertEqual(status, OrtCompiledModelCompatibility.EP_NOT_APPLICABLE) - - -if __name__ == "__main__": - unittest.main() diff --git a/onnxruntime/test/python/onnxruntime_test_python_nv_tensorrt_rtx_ep_tests.py b/onnxruntime/test/python/onnxruntime_test_python_nv_tensorrt_rtx_ep_tests.py deleted file mode 100644 index d5c80a4a1f4ba..0000000000000 --- a/onnxruntime/test/python/onnxruntime_test_python_nv_tensorrt_rtx_ep_tests.py +++ /dev/null @@ -1,468 +0,0 @@ -# Copyright (c) NVIDIA Corporation. All rights reserved. -# Licensed under the MIT License. -from __future__ import annotations - -import sys -import unittest -from collections.abc import Sequence - -import numpy as np -import torch -from autoep_helper import AutoEpTestCase -from helper import get_name -from numpy.testing import assert_almost_equal -from onnx import TensorProto, helper -from onnx.defs import onnx_opset_version - -import onnxruntime as onnxrt -from onnxruntime.capi._pybind_state import OrtDevice as C_OrtDevice -from onnxruntime.capi._pybind_state import OrtValue as C_OrtValue -from onnxruntime.capi._pybind_state import OrtValueVector, SessionIOBinding - - -class TestNvTensorRTRTXAutoEP(AutoEpTestCase): - """ - Test suite for the NvTensorRTRTX Execution Provider. - - This class contains tests for registering the NvTensorRTRTX EP, - selecting it using different policies, and running inference with various - I/O binding configurations. - """ - - ep_lib_path = "onnxruntime_providers_nv_tensorrt_rtx.dll" - ep_name = "NvTensorRTRTXExecutionProvider" - - def setUp(self): - if sys.platform != "win32": - self.skipTest("Skipping test because device discovery is only supported on Windows") - self.register_execution_provider_library(self.ep_name, self.ep_lib_path) - - def tearDown(self): - self.unregister_execution_provider_library(self.ep_name) - - def _create_ortvalue_input_on_gpu(self, device): - return onnxrt.OrtValue.ortvalue_from_numpy( - np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32), device, 0 - ) - - def _create_ortvalue_alternate_input_on_gpu(self, device): - return onnxrt.OrtValue.ortvalue_from_numpy( - np.array([[2.0, 4.0], [6.0, 8.0], [10.0, 12.0]], dtype=np.float32), - device, - 0, - ) - - def _create_uninitialized_ortvalue_input_on_gpu(self, device): - return onnxrt.OrtValue.ortvalue_from_shape_and_type([3, 2], np.float32, device, 0) - - def _create_numpy_input(self): - return np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) - - def _create_expected_output(self): - return np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) - - def _create_expected_output_alternate(self): - return np.array([[2.0, 8.0], [18.0, 32.0], [50.0, 72.0]], dtype=np.float32) - - def torch_to_onnx_type(self, torch_dtype): - if torch_dtype == torch.float32: - return TensorProto.FLOAT - elif torch_dtype == torch.float16: - return TensorProto.FLOAT16 - elif torch_dtype == torch.bfloat16: - return TensorProto.BFLOAT16 - elif torch_dtype == torch.int8: - return TensorProto.int8 - elif torch_dtype == torch.int32: - return TensorProto.INT32 - elif torch_dtype == torch.int64: - return TensorProto.INT64 - else: - raise TypeError(f"Unsupported dtype: {torch_dtype}") - - def test_nv_tensorrt_rtx_ep_register_and_inference(self): - """ - Test registration of NvTensorRTRTX EP, adding its OrtDevice to the SessionOptions, and running inference. - """ - ep_devices = onnxrt.get_ep_devices() - nv_tensorrt_rtx_ep_device = next((d for d in ep_devices if d.ep_name == self.ep_name), None) - self.assertIsNotNone(nv_tensorrt_rtx_ep_device) - self.assertEqual(nv_tensorrt_rtx_ep_device.ep_vendor, "NVIDIA") - - hw_device = nv_tensorrt_rtx_ep_device.device - self.assertEqual(hw_device.type, onnxrt.OrtHardwareDeviceType.GPU) - - # Run sample model and check output - sess = onnxrt.InferenceSession(get_name("mul_1.onnx")) - - x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) - input_name = sess.get_inputs()[0].name - res = sess.run([], {input_name: x}) - output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) - np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) - - def test_nv_tensorrt_rtx_ep_prefer_gpu_and_inference(self): - """ - Test selecting NvTensorRTRTX EP via the PREFER_GPU policy and running inference. - """ - # Set a policy to prefer GPU. NvTensorRTRTX should be selected. - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - - # Run sample model and check output - sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=sess_options) - - x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) - input_name = sess.get_inputs()[0].name - res = sess.run([], {input_name: x}) - output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) - np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) - - def test_nv_tensorrt_rtx_ep_selection_delegate_and_inference(self): - """ - Test selecting NvTensorRTRTX EP via the custom EP selection delegate function and then run inference. - """ - - # User's custom EP selection function. - def my_delegate( - ep_devices: Sequence[onnxrt.OrtEpDevice], - model_metadata: dict[str, str], - runtime_metadata: dict[str, str], - max_selections: int, - ) -> Sequence[onnxrt.OrtEpDevice]: - self.assertGreater(len(model_metadata), 0) - self.assertGreaterEqual(len(ep_devices), 1) - self.assertGreaterEqual(max_selections, 2) - - nv_tensorrt_rtx_ep_device = next((d for d in ep_devices if d.ep_name == self.ep_name), None) - self.assertIsNotNone(nv_tensorrt_rtx_ep_device) - - # Select the NvTensorRTRTX device - return [nv_tensorrt_rtx_ep_device] - - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy_delegate(my_delegate) - self.assertTrue(sess_options.has_providers()) - - # Run sample model and check output - sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=sess_options) - - x = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) - input_name = sess.get_inputs()[0].name - res = sess.run([], {input_name: x}) - output_expected = np.array([[1.0, 4.0], [9.0, 16.0], [25.0, 36.0]], dtype=np.float32) - np.testing.assert_allclose(output_expected, res[0], rtol=1e-05, atol=1e-08) - - def test_bind_input_only(self): - """ - Test I/O binding with input data only. - """ - # Set a policy to prefer GPU. NvTensorRTRTX should be selected. - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - - input = self._create_ortvalue_input_on_gpu("cuda") - - session = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=sess_options) - io_binding = session.io_binding() - - # Bind input to the GPU - io_binding.bind_input("X", "cuda", 0, np.float32, [3, 2], input.data_ptr()) - - # Sync if different streams - io_binding.synchronize_inputs() - - # Bind output to CPU - io_binding.bind_output("Y") - - # Invoke Run - session.run_with_iobinding(io_binding) - - # Sync if different streams - io_binding.synchronize_outputs() - - # Get outputs over to CPU (the outputs which were bound to the GPU will get copied over to the host - # here) - ort_output = io_binding.copy_outputs_to_cpu()[0] - - # Validate results - self.assertTrue(np.array_equal(self._create_expected_output(), ort_output)) - - def test_bind_input_and_bind_output_with_ortvalues(self): - """ - Test I/O binding with OrtValues for both input and output. - """ - # Set a policy to prefer GPU. NvTensorRTRTX EP should be selected. - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - - session = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=sess_options) - io_binding = session.io_binding() - - # Bind ortvalue as input - input_ortvalue = self._create_ortvalue_input_on_gpu("cuda") - io_binding.bind_ortvalue_input("X", input_ortvalue) - - # Bind ortvalue as output - output_ortvalue = self._create_uninitialized_ortvalue_input_on_gpu("cuda") - io_binding.bind_ortvalue_output("Y", output_ortvalue) - - # Sync if different streams - io_binding.synchronize_inputs() - - # Invoke Run - session.run_with_iobinding(io_binding) - - # Sync if different streams - io_binding.synchronize_outputs() - - # Inspect contents of output_ortvalue and make sure that it has the right contents - self.assertTrue(np.array_equal(self._create_expected_output(), output_ortvalue.numpy())) - - # Bind another ortvalue as input - input_ortvalue_2 = self._create_ortvalue_alternate_input_on_gpu("cuda") - io_binding.bind_ortvalue_input("X", input_ortvalue_2) - - # Sync if different streams - io_binding.synchronize_inputs() - - # Invoke Run - session.run_with_iobinding(io_binding) - - # Sync if different streams - io_binding.synchronize_outputs() - - # Inspect contents of output_ortvalue and make sure that it has the right contents - self.assertTrue(np.array_equal(self._create_expected_output_alternate(), output_ortvalue.numpy())) - - def test_bind_input_and_non_preallocated_output(self): - """ - Test I/O binding with non-preallocated output. - """ - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - - session = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=sess_options) - io_binding = session.io_binding() - - input = self._create_ortvalue_input_on_gpu("cuda") - - # Bind input to the GPU - io_binding.bind_input("X", "cuda", 0, np.float32, [3, 2], input.data_ptr()) - - # Bind output to the GPU - io_binding.bind_output("Y", "cuda") - - # Sync if different streams - io_binding.synchronize_inputs() - - # Invoke Run - session.run_with_iobinding(io_binding) - - # Sync if different streams - io_binding.synchronize_outputs() - - # This call returns an OrtValue which has data allocated by ORT on the GPU - ort_outputs = io_binding.get_outputs() - self.assertEqual(len(ort_outputs), 1) - self.assertEqual(ort_outputs[0].device_name(), "cuda") - # Validate results (by copying results to CPU by creating a Numpy object) - self.assertTrue(np.array_equal(self._create_expected_output(), ort_outputs[0].numpy())) - - # We should be able to repeat the above process as many times as we want - try once more - ort_outputs = io_binding.get_outputs() - self.assertEqual(len(ort_outputs), 1) - self.assertEqual(ort_outputs[0].device_name(), "cuda") - # Validate results (by copying results to CPU by creating a Numpy object) - self.assertTrue(np.array_equal(self._create_expected_output(), ort_outputs[0].numpy())) - - input = self._create_ortvalue_alternate_input_on_gpu("cuda") - - # Change the bound input and validate the results in the same bound OrtValue - # Bind alternate input to the GPU - io_binding.bind_input( - "X", - "cuda", - 0, - np.float32, - [3, 2], - input.data_ptr(), - ) - - # Sync if different streams - io_binding.synchronize_inputs() - - # Invoke Run - session.run_with_iobinding(io_binding) - - # Sync if different streams - io_binding.synchronize_outputs() - - # This call returns an OrtValue which has data allocated by ORT on the GPU - ort_outputs = io_binding.get_outputs() - self.assertEqual(len(ort_outputs), 1) - self.assertEqual(ort_outputs[0].device_name(), "cuda") - # Validate results (by copying results to CPU by creating a Numpy object) - self.assertTrue(np.array_equal(self._create_expected_output_alternate(), ort_outputs[0].numpy())) - - def test_bind_input_and_preallocated_output(self): - """ - Test I/O binding with preallocated output. - """ - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - - input = self._create_ortvalue_input_on_gpu("cuda") - - session = onnxrt.InferenceSession(get_name("mul_1.onnx"), sess_options=sess_options) - io_binding = session.io_binding() - - # Bind input to the GPU - io_binding.bind_input("X", "cuda", 0, np.float32, [3, 2], input.data_ptr()) - - # Bind output to the GPU - output = self._create_uninitialized_ortvalue_input_on_gpu("cuda") - io_binding.bind_output("Y", "cuda", 0, np.float32, [3, 2], output.data_ptr()) - - # Sync if different streams - io_binding.synchronize_inputs() - - # Invoke Run - session.run_with_iobinding(io_binding) - - # Sync if different streams - io_binding.synchronize_outputs() - - # Get outputs over to CPU (the outputs which were bound to the GPU will get copied over to the host - # here) - ort_output_vals = io_binding.copy_outputs_to_cpu()[0] - # Validate results - self.assertTrue(np.array_equal(self._create_expected_output(), ort_output_vals)) - - # Validate if ORT actually wrote to pre-allocated buffer by copying the allocated buffer - # to the host and validating its contents - ort_output_vals_in_cpu = output.numpy() - # Validate results - self.assertTrue(np.array_equal(self._create_expected_output(), ort_output_vals_in_cpu)) - - def test_bind_input_types(self): - """ - Test I/O binding with various input data types. - """ - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - opset = onnx_opset_version() - device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0) - - for dtype in [ - np.float32, - # np.float64, - np.int32, - # np.uint32, - np.int64, - # np.uint64, - # np.int16, - # np.uint16, - # np.int8, - np.uint8, - np.float16, - np.bool_, - ]: - with self.subTest(dtype=dtype, inner_device=str(device)): - x = np.arange(8).reshape((-1, 2)).astype(dtype) - proto_dtype = helper.np_dtype_to_tensor_dtype(x.dtype) - - X = helper.make_tensor_value_info("X", proto_dtype, [None, x.shape[1]]) # noqa: N806 - Y = helper.make_tensor_value_info("Y", proto_dtype, [None, x.shape[1]]) # noqa: N806 - - # inference - node_add = helper.make_node("Identity", ["X"], ["Y"]) - - # graph - graph_def = helper.make_graph([node_add], "lr", [X], [Y], []) - model_def = helper.make_model( - graph_def, - producer_name="dummy", - ir_version=7, - producer_version="0", - opset_imports=[helper.make_operatorsetid("", opset)], - ) - - sess = onnxrt.InferenceSession(model_def.SerializeToString(), sess_options=sess_options) - - bind = SessionIOBinding(sess._sess) - ort_value = C_OrtValue.ortvalue_from_numpy(x, device) - bind.bind_ortvalue_input("X", ort_value) - bind.bind_output("Y", device) - sess._sess.run_with_iobinding(bind, None) - ortvaluevector = bind.get_outputs() - self.assertIsInstance(ortvaluevector, OrtValueVector) - ortvalue = bind.get_outputs()[0] - y = ortvalue.numpy() - assert_almost_equal(x, y) - - bind = SessionIOBinding(sess._sess) - bind.bind_input("X", device, dtype, x.shape, ort_value.data_ptr()) - bind.bind_output("Y", device) - sess._sess.run_with_iobinding(bind, None) - ortvalue = bind.get_outputs()[0] - y = ortvalue.numpy() - assert_almost_equal(x, y) - - def test_bind_onnx_types_from_torch(self): - """ - Test I/O binding with various input data types. - """ - sess_options = onnxrt.SessionOptions() - sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_GPU) - self.assertTrue(sess_options.has_providers()) - opset = onnx_opset_version() - - for dtype in [ - torch.float32, - torch.float16, - torch.bfloat16, - torch.int32, - torch.int64, - ]: - with self.subTest(dtype=dtype): - proto_dtype = self.torch_to_onnx_type(dtype) - - x_ = helper.make_tensor_value_info("X", proto_dtype, [None]) - y_ = helper.make_tensor_value_info("Y", proto_dtype, [None]) - node_add = helper.make_node("Identity", ["X"], ["Y"]) - graph_def = helper.make_graph([node_add], "lr", [x_], [y_], []) - model_def = helper.make_model( - graph_def, - producer_name="dummy", - ir_version=10, - producer_version="0", - opset_imports=[helper.make_operatorsetid("", opset)], - ) - sess = onnxrt.InferenceSession(model_def.SerializeToString(), sess_options=sess_options) - - dev = "cuda" if torch.cuda.is_available() else "cpu" - device = ( - C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0) - if dev == "cuda" - else C_OrtDevice(C_OrtDevice.cpu(), C_OrtDevice.default_memory(), 0) - ) - - x = torch.arange(8, dtype=dtype, device=dev) - y = torch.empty(8, dtype=dtype, device=dev) - - bind = SessionIOBinding(sess._sess) - bind.bind_input("X", device, proto_dtype, x.shape, x.data_ptr()) - bind.bind_output("Y", device, proto_dtype, y.shape, y.data_ptr()) - sess._sess.run_with_iobinding(bind, None) - self.assertTrue(torch.equal(x, y)) - - -if __name__ == "__main__": - unittest.main(verbosity=1) diff --git a/onnxruntime/test/testdata/custom_op_library/custom_op_library.cc b/onnxruntime/test/testdata/custom_op_library/custom_op_library.cc index bc22864304567..8ab58adbeeb74 100644 --- a/onnxruntime/test/testdata/custom_op_library/custom_op_library.cc +++ b/onnxruntime/test/testdata/custom_op_library/custom_op_library.cc @@ -26,7 +26,7 @@ static void AddOrtCustomOpDomainToContainer(Ort::CustomOpDomain&& domain) { } OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api) { - Ort::InitApi(api->GetApi(ORT_API_VERSION)); + Ort::Global::api_ = api->GetApi(ORT_API_VERSION); OrtStatus* result = nullptr; ORT_TRY { diff --git a/tools/ci_build/github/windows/extract_nuget_files.ps1 b/tools/ci_build/github/windows/extract_nuget_files.ps1 index 20d6c1f2b63a5..ff8f63a85b97a 100644 --- a/tools/ci_build/github/windows/extract_nuget_files.ps1 +++ b/tools/ci_build/github/windows/extract_nuget_files.ps1 @@ -1,119 +1,105 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# This file is used by Zip-Nuget-Java Packaging Pipeline +# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline -# Define the directory for NuGet artifacts. +# Re-construct a build directory that contains binaries from all the different platforms we're including +# in the native ORT nuget package $nuget_artifacts_dir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" -# Create the directory if it doesn't exist. -New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyContinue +New-Item -Path $nuget_artifacts_dir -ItemType directory ## .zip files -# Unzip files directly, excluding the iOS xcframework to preserve its symlinks. -Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact\*" -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip | +# unzip directly +# exclude the iOS xcframework as we need to leave that zipped up to preserve symlinks +Get-ChildItem -Path $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\* -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip | Foreach-Object { - # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). - $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" - Write-Output "Executing: 7z.exe $arguments" - # Directly call 7z.exe using the call operator '&' - & 7z.exe $arguments - # Check the exit code of the last command. A non-zero code indicates an error. - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" - } + $cmd = "7z.exe x $($_.FullName) -y -o$nuget_artifacts_dir" + Write-Output $cmd + Invoke-Expression -Command $cmd } ## .tgz files -# First, extract the .tar file from the .tgz archive. -Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz | +# first extract the tar file from the tgz +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz | Foreach-Object { - # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). - $arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact", "-snld20" - Write-Output "Executing: 7z.exe $arguments" - & 7z.exe $arguments - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" - } + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" + Write-Output $cmd + Invoke-Expression -Command $cmd } -# Now, extract the contents from the .tar file into the final directory. -Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar | +# now extract the actual folder structure from the tar file to the build dir +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar | Foreach-Object { - # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). - $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" - Write-Output "Executing: 7z.exe $arguments" - & 7z.exe $arguments - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" - } + $cmd = "7z.exe x $($_.FullName) -y -o$nuget_artifacts_dir" + Write-Output $cmd + Invoke-Expression -Command $cmd } -# Process iOS xcframework -$xcframeworks = Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter onnxruntime_ios_xcframework.*.zip +# process iOS xcframework +$xcframeworks = Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter onnxruntime_ios_xcframework.*.zip if ($xcframeworks.Count -eq 1) { - $xcframework = $xcframeworks[0] - $target_dir = "$nuget_artifacts_dir\onnxruntime-ios-xcframework" - # Use the required filename format, removing version info. - $target_file = "$target_dir\onnxruntime.xcframework.zip" - New-Item -Path $target_dir -ItemType directory -ErrorAction SilentlyContinue + $xcframework = $xcframeworks[0] + $target_dir = "$nuget_artifacts_dir\onnxruntime-ios-xcframework" + # remove version info from filename and use required filename format + $target_file = "$target_dir\onnxruntime.xcframework.zip" + New-Item -Path $target_dir -ItemType directory - Write-Output "Copying $($xcframework.FullName) to $target_file" - Copy-Item $xcframework.FullName $target_file + Write-Output "Copy-Item $($xcframework.FullName) $target_file" + Copy-Item $xcframework.FullName $target_file } elseif ($xcframeworks.Count -gt 1) { - Write-Error "Expected at most one onnxruntime_ios_xcframework*.zip file but got: [$xcframeworks]" + Write-Error "Expected at most one onnxruntime_ios_xcframework*.zip file but got: [$xcframeworks]" } -# Copy Android AAR file. -# There should only be one .aar file for a full build. -$aars = Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.aar + +# copy android AAR. +# for full build of onnxruntime Android AAR, there should only be one .aar file +# called onnxruntime-android-x.y.z.aar or onnxruntime-training-android-x.y.z.aar but sanity check that +$aars = Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.aar if ($aars.Count -eq 1) { - $aar = $aars[0] - $aar_prefix = "onnxruntime" - if ($aar.Name -like "onnxruntime-training*") { - $aar_prefix = "onnxruntime-training" - } - $target_dir = "$nuget_artifacts_dir\$aar_prefix-android-aar" - # Remove version info from the filename for consistency. - $target_file = "$target_dir\onnxruntime.aar" - New-Item -Path $target_dir -ItemType directory -ErrorAction SilentlyContinue + $aar = $aars[0] + $aar_prefix = "onnxruntime" + if ($aar -like "onnxruntime-training*") { + $aar_prefix = "onnxruntime-training" + } + $target_dir = "$nuget_artifacts_dir\$aar_prefix-android-aar" + $target_file = "$target_dir\onnxruntime.aar" # remove '-mobile' and version info from filename + New-Item -Path $target_dir -ItemType directory - Write-Output "Copying $($aar.FullName) to $target_file" - Copy-Item $aar.FullName $target_file + Write-Output "Copy-Item $($aar.FullName) $target_file" + Copy-Item $aar.FullName $target_file } elseif ($aars.Count -gt 1) { - Write-Error "Expected at most one Android .aar file but got: [$aars]" + Write-Error "Expected at most one Android .aar file but got: [$aars]" } -# Check if this is a training pipeline by looking for a specific directory. -$is_training_pipeline = Test-Path -Path "$nuget_artifacts_dir\onnxruntime-training-win-x64-*" -if ($is_training_pipeline) { - Write-Output "onnxruntime-training-win-x64-* dir exists. This is a training pipeline." +# Check whether this is a training pipeline +$is_training_pipeline = $false +if (Test-Path -Path $nuget_artifacts_dir\onnxruntime-training-win-x64-*) { + $is_training_pipeline = $true + Write-Output "onnxruntime-training-win-x64-* dir exists. This is a training pipeline." } -# Copy onnxruntime and protoc binaries required by tests. -$destinationDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo" +# Copy onnxruntime and protoc binaries to the binaries dir as these are required +# by Microsoft.ML.OnnxRuntime.Tests.NetCoreApp if ($is_training_pipeline) { - Copy-Item -Path "$nuget_artifacts_dir\onnxruntime-training-win-x64-*\lib\*" -Destination $destinationDir -Recurse + Copy-Item -Path $nuget_artifacts_dir\onnxruntime-training-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo } else { - Copy-Item -Path "$nuget_artifacts_dir\onnxruntime-win-x64-*\lib\*" -Destination $destinationDir -Recurse + Copy-Item -Path $nuget_artifacts_dir\onnxruntime-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo } -# Rename directories to remove the architecture-specific suffix. -Write-Output "Renaming onnxruntime directories..." -Get-ChildItem -Directory -Path "$nuget_artifacts_dir\onnxruntime-*" | ForEach-Object { - $dirname = $_.Name - # Find the last hyphen and remove the suffix. - $lastHyphenIndex = $dirname.LastIndexOf('-') - if ($lastHyphenIndex -gt -1) { - $newName = $dirname.Substring(0, $lastHyphenIndex) - $newPath = Join-Path -Path $_.Parent.FullName -ChildPath $newName - Write-Output "Renaming '$($_.FullName)' to '$newPath'" - Rename-Item -Path $_.FullName -NewName $newName - } +"Get-ChildItem -Directory -Path $nuget_artifacts_dir\onnxruntime-*" +$ort_dirs = Get-ChildItem -Directory -Path $nuget_artifacts_dir\onnxruntime-* +foreach ($ort_dir in $ort_dirs) +{ + # remove the last '-xxx' segment from the dir name. typically that's the architecture. + $dirname = Split-Path -Path $ort_dir -Leaf + $dirname = $dirname.SubString(0,$dirname.LastIndexOf('-')) + Write-Output "Renaming $ort_dir to $dirname" + Rename-Item -Path $ort_dir -NewName $nuget_artifacts_dir\$dirname } -# List the final artifacts. -Write-Output "Post-copy artifacts:" -Get-ChildItem -Recurse $nuget_artifacts_dir \ No newline at end of file +# List artifacts +"Post copy artifacts" +Get-ChildItem -Recurse $nuget_artifacts_dir\ diff --git a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 index 29946dcb73f8a..01a8eebe75df2 100644 --- a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 +++ b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 @@ -2,81 +2,47 @@ # Licensed under the MIT License. # This file is used by Zip-Nuget-Java Packaging Pipeline -# Define the directory for NuGet artifacts. -$nuget_artifacts_dir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" -# Create the directory if it doesn't exist. -New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyContinue +New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts -ItemType directory -## .zip files -Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.zip | +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.zip | Foreach-Object { - # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). - $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" - Write-Output "Executing: 7z.exe $arguments" - & 7z.exe $arguments - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" - } + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" + Write-Output $cmd + Invoke-Expression -Command $cmd } -## .tgz files -Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz | +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz | Foreach-Object { - # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). - # *.tar will be created after *.tgz is extracted - $arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact", "-snld20" - Write-Output "Executing: 7z.exe $arguments" - & 7z.exe $arguments - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" - } + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" # *.tar will be created after *.tgz is extracted + Write-Output $cmd + Invoke-Expression -Command $cmd } -## .tar files -Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar | +Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar | Foreach-Object { - # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). - $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" - Write-Output "Executing: 7z.exe $arguments" - & 7z.exe $arguments - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" - } + $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" + Write-Output $cmd + Invoke-Expression -Command $cmd } -# Create directory for protobuf build dependencies. -New-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo" -ItemType directory -ErrorAction SilentlyContinue -# Copy CUDA libraries. -Copy-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\*" -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo" +New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo -ItemType directory + +Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo -# Install protoc via dotnet. $protocInstallDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build" dotnet new console dotnet add package Google.Protobuf.Tools --version 3.21.12 --package-directory $protocInstallDir -if ($LASTEXITCODE -ne 0) { - throw "Error adding Google.Protobuf.Tools package. Exit code: $LASTEXITCODE" -} - -# Find and copy the protoc executable. $protocDir = Get-ChildItem -Path $protocInstallDir -Recurse -Filter "protoc.exe" | Select-Object -ExpandProperty DirectoryName -First 1 -if ($protocDir) { - Write-Output "Found protoc directory: $protocDir" - Copy-Item -Path $protocDir -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo" -} -else { - Write-Error "Could not find protoc.exe in $protocInstallDir" +Write-Output $protocDir +Copy-Item -Path $protocDir -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo + +$ort_dirs = Get-ChildItem -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-* -Directory +foreach ($ort_dir in $ort_dirs) +{ + $dirname = Split-Path -Path $ort_dir -Leaf + $dirname = $dirname.SubString(0,$dirname.LastIndexOf('-')) + Write-Output "Renaming $ort_dir to $dirname" + Rename-Item -Path $ort_dir -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\$dirname } -# Rename onnxruntime directories to a generic format. -$ort_dirs = Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-*" -Directory -foreach ($ort_dir in $ort_dirs) { - $dirname = Split-Path -Path $ort_dir -Leaf - $lastHyphenIndex = $dirname.LastIndexOf('-') - if ($lastHyphenIndex -gt -1) { - $newName = $dirname.Substring(0, $lastHyphenIndex) - $newPath = Join-Path -Path $ort_dir.Parent.FullName -ChildPath $newName - Write-Output "Renaming '$($ort_dir.FullName)' to '$newPath'" - Rename-Item -Path $ort_dir.FullName -NewName $newName - } -}