diff --git a/.gitmodules b/.gitmodules index 004d8dfacb09b..7d34a0dee92d4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "cmake/external/gsl"] path = cmake/external/gsl url = https://github.com/Microsoft/GSL.git +[submodule "cmake/external/gemmlowp"] + path = cmake/external/gemmlowp + url = https://github.com/google/gemmlowp.git diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 38e9bd75cfe65..5658f0fd36042 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -374,29 +374,6 @@ else() endif() endif() -if (onnxruntime_USE_TVM) - if (WIN32 AND MSVC) - # wd4100: identifier' : unreferenced formal parameter - # wd4244: conversion from 'int' to 'char', possible loss of data - # wd4251: class X needs to have dll-interface to be used by clients of class Y - # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data - # wd4275: non dll-interface class X used as base for dll-interface class Y - # wd4389: signed/unsigned mismatch - # wd4456: declaration of X hides previous local declaration - set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") - else() - set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers") - if(HAS_UNUSED_PARAMETER) - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") - endif() - if(HAS_CATCH_VALUE) - #TODO: send a PR to TVM and fix it - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") - endif() - endif() - include(onnxruntime_codegen.cmake) -endif() - if (onnxruntime_USE_JEMALLOC) if (Win32) message( FATAL_ERROR "Jemalloc is not supported on Windows." ) @@ -412,11 +389,27 @@ include_directories( $ ) +if (onnxruntime_USE_MKLDNN OR onnxruntime_USE_MKLML) + include(mkldnn) +endif() + +if (onnxruntime_USE_MKLML) + add_definitions(-DUSE_MKLML=1) + # USE_MKML_FOR_BLAS may cause numerical differences in tests so disable by default + #add_definitions(-DUSE_MKLML_FOR_BLAS=1) + if (WIN32 OR APPLE) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml) + else() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel) + endif() + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mklml) + link_directories(${MKLML_LIB_DIR}) +endif() + if (onnxruntime_USE_MKLDNN) add_definitions(-DUSE_MKLDNN=1) - include(mkldnn) list(APPEND onnxruntime_EXTERNAL_LIBRARIES mkldnn) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mkldnn) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mkldnn) link_directories(${MKLDNN_LIB_DIR}) endif() @@ -442,7 +435,6 @@ if (onnxruntime_USE_CUDA) endif () file(TO_CMAKE_PATH ${onnxruntime_CUDNN_HOME} onnxruntime_CUDNN_HOME) find_package(CUDA 9.0 REQUIRED) - include(cub) set(CUDA_LINK_LIBRARIES_KEYWORD PRIVATE) if (WIN32) link_directories(${onnxruntime_CUDNN_HOME}/lib/x64) @@ -453,7 +445,6 @@ if (onnxruntime_USE_CUDA) set(ONNXRUNTIME_CUDA_LIBRARIES cudnn_static cublas_static culibos) endif() list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ONNXRUNTIME_CUDA_LIBRARIES}) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES cub) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_30,code=sm_30") # K series set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_50,code=sm_50") # M series @@ -465,6 +456,33 @@ if (onnxruntime_USE_CUDA) endif() endif() +if (onnxruntime_USE_TVM) + if (WIN32 AND MSVC) + # wd4100: identifier' : unreferenced formal parameter + # wd4244: conversion from 'int' to 'char', possible loss of data + # wd4251: class X needs to have dll-interface to be used by clients of class Y + # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data + # wd4275: non dll-interface class X used as base for dll-interface class Y + # wd4389: signed/unsigned mismatch + # wd4456: declaration of X hides previous local declaration + set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") + else() + set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers") + if(HAS_UNUSED_PARAMETER) + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") + endif() + if(HAS_CATCH_VALUE) + #TODO: send a PR to TVM and fix it + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") + endif() + endif() + include(onnxruntime_codegen.cmake) +endif() + +if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) + add_definitions(-DMICROSOFT_INTERNAL) +endif() + #names in this var must match the directory names under onnxruntime/core/providers set(ONNXRUNTIME_PROVIDER_NAMES cpu) diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake deleted file mode 100644 index 080e11c7fd22e..0000000000000 --- a/cmake/external/cub.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(cub_URL https://github.com/NVlabs/cub/archive/v1.8.0.zip) -set(cub_HASH SHA256=6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3) -set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) -set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) -set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive) - -ExternalProject_Add(cub - PREFIX cub - URL ${cub_URL} - URL_HASH ${cub_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/cub/CMakeLists.txt ${cub_BUILD} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${cub_INCLUDE_DIR}/cub ${cub_ARCHIVE_DIR}/cub) diff --git a/cmake/external/gemmlowp b/cmake/external/gemmlowp new file mode 160000 index 0000000000000..a3df028932a6b --- /dev/null +++ b/cmake/external/gemmlowp @@ -0,0 +1 @@ +Subproject commit a3df028932a6b00e6ef9c4b6f1c6109f696c915d diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index a789382bf80ee..72ed6a108d939 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -1,57 +1,73 @@ include (ExternalProject) set(MKLDNN_URL https://github.com/intel/mkl-dnn.git) -# If MKLDNN_TAG is updated, check if platform.cmake.patch needs to be updated. -set(MKLDNN_TAG v0.17.1) -set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src) -set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install) -set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib) -set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include) - -set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch) -# discard prior changes due to patching in mkldnn source to unblock incremental builds. -set(MKLDNN_PATCH_DISCARD_COMMAND cd ${MKLDNN_SOURCE} && git checkout -- .) +# If MKLDNN_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated. +set(MKLDNN_TAG v0.17.2) +set(MKLML_VERSION 2019.0.1.20180928) if(WIN32) + set(MKLML_OS_VERSION_STR "win") + set(MKLML_FILE_EXTENSION "zip") set(MKLDNN_SHARED_LIB mkldnn.dll) set(MKLDNN_IMPORT_LIB mkldnn.lib) if(onnxruntime_USE_MKLML) - set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.bat) set(MKLML_SHARED_LIB mklml.dll) + set(MKLML_IMPORT_LIB mklml.lib) set(IOMP5MD_SHARED_LIB libiomp5md.dll) + set(IOMP5MD_IMPORT_LIB libiomp5md.lib) endif() else() + set(MKLML_FILE_EXTENSION "tgz") if (APPLE) set(MKLDNN_SHARED_LIB libmkldnn.0.dylib) + set(MKLML_OS_VERSION_STR "mac") else() set(MKLDNN_SHARED_LIB libmkldnn.so.0) + set(MKLML_OS_VERSION_STR "lnx") endif() if(onnxruntime_USE_MKLML) - set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.sh) set(MKLML_SHARED_LIB libmklml_intel.so) set(IOMP5MD_SHARED_LIB libiomp5.so) endif() endif() -if(NOT onnxruntime_USE_MKLDNN OR EXISTS ${MKLDNN_SOURCE}/external) - set(DOWNLOAD_MKLML "") +if (onnxruntime_USE_MKLML) + set(MKLML_URL https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_TAG}/mklml_${MKLML_OS_VERSION_STR}_${MKLML_VERSION}.${MKLML_FILE_EXTENSION}) + + ExternalProject_Add(project_mklml + PREFIX mklml + URL ${MKLML_URL} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND "" ) + + set(MKML_DIR ${CMAKE_CURRENT_BINARY_DIR}/mklml/src/project_mklml) + set(MKLML_INCLUDE_DIR "${MKML_DIR}/include") + set(MKLML_LIB_DIR "${MKML_DIR}/lib") + link_directories(${MKLML_LIB_DIR}) endif() -ExternalProject_Add(project_mkldnn +if (onnxruntime_USE_MKLDNN) + set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src) + set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install) + set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib) + set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include) + + set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch) + # discard prior changes due to patching in mkldnn source to unblock incremental builds. + set(MKLDNN_PATCH_DISCARD_COMMAND cd ${MKLDNN_SOURCE} && git checkout -- .) + + ExternalProject_Add(project_mkldnn PREFIX mkl-dnn GIT_REPOSITORY ${MKLDNN_URL} GIT_TAG ${MKLDNN_TAG} - PATCH_COMMAND ${DOWNLOAD_MKLML} COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${MKLDNN_PATCH_COMMAND1} + PATCH_COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${MKLDNN_PATCH_COMMAND1} SOURCE_DIR ${MKLDNN_SOURCE} - CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} -) - -if(WIN32) - add_library(mkldnn STATIC IMPORTED) - set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB}) -else() - add_library(mkldnn SHARED IMPORTED) - set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB}) + CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} -DMKLROOT=${MKML_DIR} + ) + link_directories(${MKLDNN_LIB_DIR}) + if (onnxruntime_USE_MKLML) + add_dependencies(project_mkldnn project_mklml) + endif() endif() -add_dependencies(mkldnn project_mkldnn) -include_directories(${MKLDNN_INCLUDE_DIR}) diff --git a/cmake/external/onnx b/cmake/external/onnx index 0a4d5abdf4939..0c8d857bb1624 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit 0a4d5abdf4939ab0842a5eadcc16a3bf0738f901 +Subproject commit 0c8d857bb162431912b255d5c0e773fb7c131a65 diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 79734f8a1315d..db74da04746ca 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -33,7 +33,7 @@ source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_contrib_ops_srcs}) add_library(onnxruntime_providers ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs} ${onnxruntime_contrib_ops_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers onnx protobuf::libprotobuf) target_include_directories(onnxruntime_providers PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) -add_dependencies(onnxruntime_providers eigen gsl onnx) +add_dependencies(onnxruntime_providers eigen gsl onnx ${onnxruntime_EXTERNAL_DEPENDENCIES}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cpu DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers PROPERTIES FOLDER "ONNXRuntime") @@ -94,7 +94,7 @@ if (onnxruntime_USE_MKLDNN) onnxruntime_add_include_to_target(onnxruntime_providers_mkldnn onnx protobuf::libprotobuf) add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime") - target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) + target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${MKLDNN_INCLUDE_DIR} ${MKLML_INCLUDE_DIR}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/mkldnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX) endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 22db4d45f2ea3..0de14f9333dc1 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -182,7 +182,7 @@ if (onnxruntime_USE_MKLML) add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB} + ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} $/onnxruntime/capi/ ) endif() diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index c11ab46ad9fae..c1a1393bcc973 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -437,7 +437,7 @@ if (onnxruntime_USE_MKLML) add_custom_command( TARGET ${test_data_target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB} + ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} $ ) endif() diff --git a/cmake/onnxruntime_util.cmake b/cmake/onnxruntime_util.cmake index 525d851a92d70..af1b51326f1bc 100644 --- a/cmake/onnxruntime_util.cmake +++ b/cmake/onnxruntime_util.cmake @@ -9,7 +9,7 @@ file(GLOB_RECURSE onnxruntime_util_srcs source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs}) add_library(onnxruntime_util ${onnxruntime_util_srcs}) -target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) +target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${MKLDNN_INCLUDE_DIR} ${MKLML_INCLUDE_DIR}) onnxruntime_add_include_to_target(onnxruntime_util onnx protobuf::libprotobuf) set_target_properties(onnxruntime_util PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_util PROPERTIES FOLDER "ONNXRuntime") diff --git a/cmake/patches/cub/CMakeLists.txt b/cmake/patches/cub/CMakeLists.txt deleted file mode 100644 index 8fbb114c345bd..0000000000000 --- a/cmake/patches/cub/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -cmake_minimum_required(VERSION 2.8.3) - -project(cub) diff --git a/cmake/patches/protobuf.patch b/cmake/patches/protobuf.patch deleted file mode 100644 index 111d5655ca0d2..0000000000000 --- a/cmake/patches/protobuf.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc -index a066a6a7..636a864f 100644 ---- a/src/google/protobuf/compiler/cpp/cpp_file.cc -+++ b/src/google/protobuf/compiler/cpp/cpp_file.cc -@@ -972,6 +972,11 @@ void FileGenerator::GenerateTopHeaderGuard(io::Printer* printer, - "#ifndef PROTOBUF_$filename_identifier$__INCLUDED\n" - "#define PROTOBUF_$filename_identifier$__INCLUDED\n" - "\n" -+ "#ifdef _MSC_VER\n" -+ "#pragma warning(push)\n" -+ "#pragma warning(disable: 4800)\n" -+ "#endif // _MSC_VER\n" -+ "\n" - "#include \n", - "filename", file_->name(), "filename_identifier", filename_identifier); - printer->Print("\n"); -@@ -980,6 +985,10 @@ void FileGenerator::GenerateTopHeaderGuard(io::Printer* printer, - void FileGenerator::GenerateBottomHeaderGuard( - io::Printer* printer, const string& filename_identifier) { - printer->Print( -+ "#ifdef _MSC_VER\n" -+ "#pragma warning(pop)\n" -+ "#endif // _MSC_VER\n" -+ "\n" - "#endif // PROTOBUF_$filename_identifier$__INCLUDED\n", - "filename_identifier", filename_identifier); - } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs index c7698722d8727..f748ffe08035d 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs @@ -26,7 +26,6 @@ internal enum ErrorCode InvalidGraph = 10, ShapeInferenceNotRegistered = 11, RequirementNotRegistered = 12, - ExecutionProviderDLLNotFound = 13 } /// @@ -49,7 +48,6 @@ public class OnnxRuntimeException: Exception { ErrorCode.InvalidGraph, "InvalidGraph" }, { ErrorCode.ShapeInferenceNotRegistered, "ShapeInferenceNotRegistered" }, { ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" }, - { ErrorCode.ExecutionProviderDLLNotFound, "ExecutionProviderDLLNotFound" } }; internal OnnxRuntimeException(ErrorCode errorCode, string message) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index c3382f3d4b3dc..8973a982d1094 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -74,7 +74,7 @@ public InferenceSession(string modelPath, SessionOptions options) { if (_nativeHandle != IntPtr.Zero) { - NativeMethods.ReleaseONNXSession(_nativeHandle); + NativeMethods.OrtReleaseSession(_nativeHandle); _nativeHandle = IntPtr.Zero; } throw e; @@ -178,7 +178,7 @@ internal IReadOnlyCollection Run(IReadOnlyCollection Run(IReadOnlyCollectiontrue true false - OnnxRuntime.snk + OnnxRuntime.snk Microsoft.ML.OnnxRuntime @@ -31,6 +31,18 @@ + + - - + @(MajorVersionNumber) $(PackageVersion) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs index 55b609be8cb11..19eb93e662555 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs @@ -161,7 +161,7 @@ out nativeElementType longShape[i] = (ulong)shape[i]; } - IntPtr status = NativeMethods.OrtCreateTensorWithDataAsONNXValue( + IntPtr status = NativeMethods.OrtCreateTensorWithDataAsOrtValue( NativeMemoryAllocatorInfo.DefaultInstance.Handle, dataBufferPointer, (ulong)(dataBufferLength), diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs index f7696c725ebb9..dabe0b4d68811 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs @@ -27,7 +27,7 @@ public static void VerifySuccess(IntPtr nativeStatus) { ErrorCode statusCode = NativeMethods.OrtGetErrorCode(nativeStatus); string errorMessage = GetErrorMessage(nativeStatus); - NativeMethods.ReleaseONNXStatus(nativeStatus); + NativeMethods.OrtReleaseStatus(nativeStatus); throw new OnnxRuntimeException(statusCode, errorMessage); } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 861e717168614..c668b7e905095 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -17,44 +17,44 @@ internal static class NativeMethods #region Runtime/Environment API [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* ONNXStatus* */OrtInitialize( + public static extern IntPtr /* OrtStatus* */OrtInitialize( LogLevel default_warning_level, string logId, - out IntPtr /*(ONNXEnv*)*/ env); - // ReleaseONNXEnv should not be used + out IntPtr /*(OrtEnv*)*/ env); + // OrtReleaseEnv should not be used [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXEnv(IntPtr /*(ONNXEnv*)*/ env); + public static extern void OrtReleaseEnv(IntPtr /*(OrtEnv*)*/ env); #endregion Runtime/Environment API #region Status API [DllImport(nativeLib, CharSet = charSet)] - public static extern ErrorCode OrtGetErrorCode(IntPtr /*(ONNXStatus*)*/status); + public static extern ErrorCode OrtGetErrorCode(IntPtr /*(OrtStatus*)*/status); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* char* */OrtGetErrorMessage(IntPtr /* (ONNXStatus*) */status); + public static extern IntPtr /* char* */OrtGetErrorMessage(IntPtr /* (OrtStatus*) */status); // returns char*, need to convert to string by the caller. - // does not free the underlying ONNXStatus* + // does not free the underlying OrtStatus* [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXStatus(IntPtr /*(ONNXStatus*)*/ statusPtr); + public static extern void OrtReleaseStatus(IntPtr /*(OrtStatus*)*/ statusPtr); #endregion Status API #region InferenceSession API [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* ONNXStatus* */OrtCreateInferenceSession( - IntPtr /* (ONNXEnv*) */ environment, + public static extern IntPtr /* OrtStatus* */OrtCreateInferenceSession( + IntPtr /* (OrtEnv*) */ environment, [MarshalAs(UnmanagedType.LPWStr)]string modelPath, //the model path is consumed as a wchar* in the C-api IntPtr /* (OrtSessionOptions*) */sessopnOptions, out IntPtr /**/ session); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNStatus*)*/ OrtRunInference( - IntPtr /*(ONNXSession*)*/ session, - IntPtr /*(ONNXSessionRunOptions*)*/ runOptions, // can be null to use the default options + IntPtr /*(OrtSession*)*/ session, + IntPtr /*(OrtSessionRunOptions*)*/ runOptions, // can be null to use the default options string[] inputNames, - IntPtr[] /* (ONNXValue*[])*/ inputValues, + IntPtr[] /* (OrtValue*[])*/ inputValues, ulong inputCount, /* TODO: size_t, make it portable for x86 arm */ string[] outputNames, ulong outputCount, /* TODO: size_t, make it portable for x86 and arm */ @@ -65,47 +65,47 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtInferenceSessionGetInputCount( - IntPtr /*(ONNXSession*)*/ session, + public static extern IntPtr /*(OrtStatus*)*/ OrtInferenceSessionGetInputCount( + IntPtr /*(OrtSession*)*/ session, out ulong /* TODO: size_t */ count); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtInferenceSessionGetOutputCount( - IntPtr /*(ONNXSession*)*/ session, + public static extern IntPtr /*(OrtStatus*)*/ OrtInferenceSessionGetOutputCount( + IntPtr /*(OrtSession*)*/ session, out ulong /*TODO: size_t port*/ count); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/OrtInferenceSessionGetInputName( - IntPtr /*(ONNXSession*)*/ session, + public static extern IntPtr /*(OrtStatus*)*/OrtInferenceSessionGetInputName( + IntPtr /*(OrtSession*)*/ session, ulong index, //TODO: port size_t IntPtr /*(OrtAllocator*)*/ allocator, out IntPtr /*(char**)*/name); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/OrtInferenceSessionGetOutputName( - IntPtr /*(ONNXSession*)*/ session, + public static extern IntPtr /*(OrtStatus*)*/OrtInferenceSessionGetOutputName( + IntPtr /*(OrtSession*)*/ session, ulong index, //TODO: port size_t IntPtr /*(OrtAllocator*)*/ allocator, out IntPtr /*(char**)*/name); // release the typeinfo using OrtReleaseObject [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/OrtInferenceSessionGetInputTypeInfo( - IntPtr /*(const ONNXSession*)*/ session, + public static extern IntPtr /*(OrtStatus*)*/OrtInferenceSessionGetInputTypeInfo( + IntPtr /*(const OrtSession*)*/ session, ulong index, //TODO: port for size_t out IntPtr /*(struct OrtTypeInfo**)*/ typeInfo); // release the typeinfo using OrtReleaseObject [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/OrtInferenceSessionGetOutputTypeInfo( - IntPtr /*(const ONNXSession*)*/ session, + public static extern IntPtr /*(OrtStatus*)*/OrtInferenceSessionGetOutputTypeInfo( + IntPtr /*(const OrtSession*)*/ session, ulong index, //TODO: port for size_t out IntPtr /* (struct OrtTypeInfo**)*/ typeInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXSession(IntPtr /*(ONNXSession*)*/session); + public static extern void OrtReleaseSession(IntPtr /*(OrtSession*)*/session); #endregion InferenceSession API @@ -161,16 +161,16 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca public static extern void OrtSessionOptionsAppendExecutionProvider(IntPtr /*(OrtSessionOptions*)*/ options, IntPtr /* (OrtProviderFactoryPtr*)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtCreateCpuExecutionProviderFactory(int use_arena, out IntPtr /*(OrtProviderFactoryPtr*)*/ factory); + public static extern IntPtr /*(OrtStatus*)*/ OrtCreateCpuExecutionProviderFactory(int use_arena, out IntPtr /*(OrtProviderFactoryPtr*)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtCreateMkldnnExecutionProviderFactory(int use_arena, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory); + public static extern IntPtr /*(OrtStatus*)*/ OrtCreateMkldnnExecutionProviderFactory(int use_arena, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtCreateCUDAExecutionProviderFactory(int device_id, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory); + public static extern IntPtr /*(OrtStatus*)*/ OrtCreateCUDAExecutionProviderFactory(int device_id, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtCreateNupharExecutionProviderFactory(int device_id, string target_str, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory); + public static extern IntPtr /*(OrtStatus*)*/ OrtCreateNupharExecutionProviderFactory(int device_id, string target_str, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path); @@ -197,7 +197,7 @@ public enum MemoryType [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* (ONNXStatus*)*/ OrtCreateAllocatorInfo( + public static extern IntPtr /* (OrtStatus*)*/ OrtCreateAllocatorInfo( IntPtr /*(const char*) */name, AllocatorType allocatorType, int identifier, @@ -205,9 +205,9 @@ public enum MemoryType out IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo // memory ownership transfered to caller ); - //ONNXRUNTIME_API_STATUS(OrtCreateCpuAllocatorInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out) + //ORT_API_STATUS(OrtCreateCpuAllocatorInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out) [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* (ONNXStatus*)*/ OrtCreateCpuAllocatorInfo( + public static extern IntPtr /* (OrtStatus*)*/ OrtCreateCpuAllocatorInfo( AllocatorType allocatorType, MemoryType memoryType, out IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo @@ -217,7 +217,7 @@ public enum MemoryType public static extern void ReleaseOrtAllocatorInfo(IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/OrtCreateDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); + public static extern IntPtr /*(OrtStatus*)*/OrtCreateDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); /// /// Releases/Unrefs any object, including the Allocator @@ -243,32 +243,32 @@ public enum MemoryType #region Tensor/OnnxValue API [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* ONNXStatus */ OrtCreateTensorWithDataAsONNXValue( + public static extern IntPtr /* OrtStatus */ OrtCreateTensorWithDataAsOrtValue( IntPtr /* (const OrtAllocatorInfo*) */ allocatorInfo, IntPtr /* (void*) */dataBufferHandle, ulong dataLength, //size_t, TODO: make it portable for x86, arm ulong[] shape, //size_t* or size_t[], TODO: make it portable for x86, arm ulong shapeLength, //size_t, TODO: make it portable for x86, arm TensorElementType type, - out IntPtr /* ONNXValuePtr* */ outputValue); + out IntPtr /* OrtValue** */ outputValue); /// This function doesn't work with string tensor - /// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. + /// this is a no-copy method whose pointer is only valid until the backing OrtValue* is free'd. [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorMutableData(IntPtr /*(ONNXValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); + public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorMutableData(IntPtr /*(OrtValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); //[DllImport(nativeLib, CharSet = charSet)] - //public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorShapeDimCount(IntPtr /*(ONNXValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm + //public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorShapeDimCount(IntPtr /*(OrtValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm //[DllImport(nativeLib, CharSet = charSet)] - //public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count); + //public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorShapeElementCount(IntPtr /*(OrtValue*)*/value, out ulong count); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ OrtCastTypeInfoToTensorInfo(IntPtr /*(struct OrtTypeInfo*)*/ typeInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorShapeAndType(IntPtr /*(ONNXValue*)*/ value, out IntPtr /*(struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); + public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorShapeAndType(IntPtr /*(OrtValue*)*/ value, out IntPtr /*(struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern TensorElementType OrtGetTensorElementType(IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); @@ -295,7 +295,7 @@ public static extern void OrtGetDimensions( public static extern long OrtGetTensorShapeElementCount(IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value); + public static extern void OrtReleaseValue(IntPtr /*(OrtValue*)*/ value); #endregion } //class NativeMethods diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs index d48d097240a5e..6b4e22fb71dd5 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs @@ -191,7 +191,7 @@ protected override void Dispose(bool disposing) // do managed objects cleanup } - NativeMethods.ReleaseONNXValue(_onnxValueHandle); + NativeMethods.OrtReleaseValue(_onnxValueHandle); _disposed = true; } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs index da46fc8a8bb13..1d3abcbf0d8aa 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs @@ -70,7 +70,7 @@ public override bool IsInvalid private static void Delete(IntPtr nativePtr) { - NativeMethods.ReleaseONNXEnv(nativePtr); + NativeMethods.OrtReleaseEnv(nativePtr); } protected override bool ReleaseHandle() diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index b1d69104cb188..0e8f88c6dabe9 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Text; using System.Runtime.InteropServices; namespace Microsoft.ML.OnnxRuntime @@ -125,9 +126,12 @@ private void AppendExecutionProvider(NativeOnnxObjectHandle providerFactory) } } + // Declared, but called only if OS = Windows. [DllImport("kernel32.dll")] private static extern IntPtr LoadLibrary(string dllToLoad); + [DllImport("kernel32.dll")] + static extern uint GetSystemDirectory([Out] StringBuilder lpBuffer, uint uSize); private static bool CheckCudaExecutionProviderDLLs() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -135,14 +139,15 @@ private static bool CheckCudaExecutionProviderDLLs() foreach (var dll in cudaDelayLoadedLibs) { IntPtr handle = LoadLibrary(dll); - if (handle == IntPtr.Zero) - { - throw new OnnxRuntimeException( - ErrorCode.ExecutionProviderDLLNotFound, - $"Dll not found: {dll}. CUDA 10.0 is required for GPU execution. " + - $"Verify that the library is available on system path." - ); - } + if (handle != IntPtr.Zero) + continue; + var sysdir = new StringBuilder(String.Empty, 2048); + GetSystemDirectory(sysdir, (uint)sysdir.Capacity); + throw new OnnxRuntimeException( + ErrorCode.NoSuchFile, + $"kernel32.LoadLibrary():'{dll}' not found. CUDA 10.0 is required for GPU execution. " + + $". Verify it is available in the system directory={sysdir}. Else copy it to the output folder." + ); } } return true; diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 995c6acbb91d1..256792f7343da 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -212,30 +212,36 @@ private void TestPreTrainedModelsOpset7And8() foreach (var opset in opsets) { var modelRoot = new DirectoryInfo(opset); - foreach (var model in modelRoot.EnumerateDirectories()) + foreach (var modelDir in modelRoot.EnumerateDirectories()) { // TODO: dims contains 'None'. Session throws error. - if (model.ToString() == "test_tiny_yolov2") + if (modelDir.Name== "test_tiny_yolov2") continue; + + String onnxModelFileName = null; try { - var modelNames = model.GetFiles("*.onnx"); - if (modelNames.Count() != 1) + var onnxModelNames = modelDir.GetFiles("*.onnx"); + if (onnxModelNames.Count() != 1) { // TODO remove file "._resnet34v2.onnx" from test set - if (modelNames[0].ToString() == "._resnet34v2.onnx") - modelNames[0] = modelNames[1]; + if (onnxModelNames[0].Name == "._resnet34v2.onnx") + onnxModelNames[0] = onnxModelNames[1]; else - throw new Exception($"Opset {opset}: Model {model}: error = can't determine model file name."); + { + var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString())); + throw new Exception($"Opset {opset}: Model {modelDir}. Can't determine model file name. Found these :{modelNamesList}"); + } } - var session = new InferenceSession($"{opset}\\{model}\\{modelNames[0].ToString()}"); + onnxModelFileName = $"{opset}\\{modelDir.Name}\\{onnxModelNames[0].Name}"; + var session = new InferenceSession(onnxModelFileName); var inMeta = session.InputMetadata; var innodepair = inMeta.First(); var innodename = innodepair.Key; var innodedims = innodepair.Value.Dimensions; - var dataIn = LoadTensorFromFilePb($"{opset}\\{model}\\test_data_set_0\\input_0.pb"); - var dataOut = LoadTensorFromFilePb($"{opset}\\{model}\\test_data_set_0\\output_0.pb"); + var dataIn = LoadTensorFromFilePb($"{opset}\\{modelDir.Name}\\test_data_set_0\\input_0.pb"); + var dataOut = LoadTensorFromFilePb($"{opset}\\{modelDir.Name}\\test_data_set_0\\output_0.pb"); var tensorIn = new DenseTensor(dataIn, innodedims); var nov = new List(); nov.Add(NamedOnnxValue.CreateFromTensor(innodename, tensorIn)); @@ -246,7 +252,7 @@ private void TestPreTrainedModelsOpset7And8() } catch (Exception ex) { - var msg = $"Opset {opset}: Model {model}: error = {ex.Message}"; + var msg = $"Opset {opset}: Model {modelDir}: ModelFile = {onnxModelFileName} error = {ex.Message}"; throw new Exception(msg); } } //model diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj index ac3702b7e3df5..252d3ba783874 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj @@ -1,4 +1,4 @@ - + netcoreapp2.0 @@ -38,8 +38,8 @@ false - - Always + + PreserveNewest false diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs index baf8f0e80e216..209e62c30a5f5 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs @@ -53,41 +53,39 @@ static OnnxMlReflection() { "b3RvEhIKCmRvY19zdHJpbmcYCiABKAkSIwoFaW5wdXQYCyADKAsyFC5vbm54", "LlZhbHVlSW5mb1Byb3RvEiQKBm91dHB1dBgMIAMoCzIULm9ubnguVmFsdWVJ", "bmZvUHJvdG8SKAoKdmFsdWVfaW5mbxgNIAMoCzIULm9ubnguVmFsdWVJbmZv", - "UHJvdG8ivQQKC1RlbnNvclByb3RvEgwKBGRpbXMYASADKAMSLQoJZGF0YV90", - "eXBlGAIgASgOMhoub25ueC5UZW5zb3JQcm90by5EYXRhVHlwZRIqCgdzZWdt", - "ZW50GAMgASgLMhkub25ueC5UZW5zb3JQcm90by5TZWdtZW50EhYKCmZsb2F0", - "X2RhdGEYBCADKAJCAhABEhYKCmludDMyX2RhdGEYBSADKAVCAhABEhMKC3N0", - "cmluZ19kYXRhGAYgAygMEhYKCmludDY0X2RhdGEYByADKANCAhABEgwKBG5h", - "bWUYCCABKAkSEgoKZG9jX3N0cmluZxgMIAEoCRIQCghyYXdfZGF0YRgJIAEo", - "DBIXCgtkb3VibGVfZGF0YRgKIAMoAUICEAESFwoLdWludDY0X2RhdGEYCyAD", - "KARCAhABGiUKB1NlZ21lbnQSDQoFYmVnaW4YASABKAMSCwoDZW5kGAIgASgD", - "ItoBCghEYXRhVHlwZRINCglVTkRFRklORUQQABIJCgVGTE9BVBABEgkKBVVJ", - "TlQ4EAISCAoESU5UOBADEgoKBlVJTlQxNhAEEgkKBUlOVDE2EAUSCQoFSU5U", - "MzIQBhIJCgVJTlQ2NBAHEgoKBlNUUklORxAIEggKBEJPT0wQCRILCgdGTE9B", - "VDE2EAoSCgoGRE9VQkxFEAsSCgoGVUlOVDMyEAwSCgoGVUlOVDY0EA0SDQoJ", - "Q09NUExFWDY0EA4SDgoKQ09NUExFWDEyOBAPEgwKCEJGTE9BVDE2EBAilQEK", - "EFRlbnNvclNoYXBlUHJvdG8SLQoDZGltGAEgAygLMiAub25ueC5UZW5zb3JT", - "aGFwZVByb3RvLkRpbWVuc2lvbhpSCglEaW1lbnNpb24SEwoJZGltX3ZhbHVl", - "GAEgASgDSAASEwoJZGltX3BhcmFtGAIgASgJSAASEgoKZGVub3RhdGlvbhgD", - "IAEoCUIHCgV2YWx1ZSKWBQoJVHlwZVByb3RvEi0KC3RlbnNvcl90eXBlGAEg", - "ASgLMhYub25ueC5UeXBlUHJvdG8uVGVuc29ySAASMQoNc2VxdWVuY2VfdHlw", - "ZRgEIAEoCzIYLm9ubnguVHlwZVByb3RvLlNlcXVlbmNlSAASJwoIbWFwX3R5", - "cGUYBSABKAsyEy5vbm54LlR5cGVQcm90by5NYXBIABItCgtvcGFxdWVfdHlw", - "ZRgHIAEoCzIWLm9ubnguVHlwZVByb3RvLk9wYXF1ZUgAEjoKEnNwYXJzZV90", - "ZW5zb3JfdHlwZRgIIAEoCzIcLm9ubnguVHlwZVByb3RvLlNwYXJzZVRlbnNv", - "ckgAEhIKCmRlbm90YXRpb24YBiABKAkaXgoGVGVuc29yEi0KCWVsZW1fdHlw", - "ZRgBIAEoDjIaLm9ubnguVGVuc29yUHJvdG8uRGF0YVR5cGUSJQoFc2hhcGUY", - "AiABKAsyFi5vbm54LlRlbnNvclNoYXBlUHJvdG8aLgoIU2VxdWVuY2USIgoJ", - "ZWxlbV90eXBlGAEgASgLMg8ub25ueC5UeXBlUHJvdG8aWAoDTWFwEiwKCGtl", - "eV90eXBlGAEgASgOMhoub25ueC5UZW5zb3JQcm90by5EYXRhVHlwZRIjCgp2", - "YWx1ZV90eXBlGAIgASgLMg8ub25ueC5UeXBlUHJvdG8aJgoGT3BhcXVlEg4K", - "BmRvbWFpbhgBIAEoCRIMCgRuYW1lGAIgASgJGmQKDFNwYXJzZVRlbnNvchIt", - "CgllbGVtX3R5cGUYASABKA4yGi5vbm54LlRlbnNvclByb3RvLkRhdGFUeXBl", - "EiUKBXNoYXBlGAIgASgLMhYub25ueC5UZW5zb3JTaGFwZVByb3RvQgcKBXZh", - "bHVlIjUKEk9wZXJhdG9yU2V0SWRQcm90bxIOCgZkb21haW4YASABKAkSDwoH", - "dmVyc2lvbhgCIAEoAypjCgdWZXJzaW9uEhIKDl9TVEFSVF9WRVJTSU9OEAAS", - "GQoVSVJfVkVSU0lPTl8yMDE3XzEwXzEwEAESGQoVSVJfVkVSU0lPTl8yMDE3", - "XzEwXzMwEAISDgoKSVJfVkVSU0lPThADYgZwcm90bzM=")); + "UHJvdG8ioQQKC1RlbnNvclByb3RvEgwKBGRpbXMYASADKAMSEQoJZGF0YV90", + "eXBlGAIgASgFEioKB3NlZ21lbnQYAyABKAsyGS5vbm54LlRlbnNvclByb3Rv", + "LlNlZ21lbnQSFgoKZmxvYXRfZGF0YRgEIAMoAkICEAESFgoKaW50MzJfZGF0", + "YRgFIAMoBUICEAESEwoLc3RyaW5nX2RhdGEYBiADKAwSFgoKaW50NjRfZGF0", + "YRgHIAMoA0ICEAESDAoEbmFtZRgIIAEoCRISCgpkb2Nfc3RyaW5nGAwgASgJ", + "EhAKCHJhd19kYXRhGAkgASgMEhcKC2RvdWJsZV9kYXRhGAogAygBQgIQARIX", + "Cgt1aW50NjRfZGF0YRgLIAMoBEICEAEaJQoHU2VnbWVudBINCgViZWdpbhgB", + "IAEoAxILCgNlbmQYAiABKAMi2gEKCERhdGFUeXBlEg0KCVVOREVGSU5FRBAA", + "EgkKBUZMT0FUEAESCQoFVUlOVDgQAhIICgRJTlQ4EAMSCgoGVUlOVDE2EAQS", + "CQoFSU5UMTYQBRIJCgVJTlQzMhAGEgkKBUlOVDY0EAcSCgoGU1RSSU5HEAgS", + "CAoEQk9PTBAJEgsKB0ZMT0FUMTYQChIKCgZET1VCTEUQCxIKCgZVSU5UMzIQ", + "DBIKCgZVSU5UNjQQDRINCglDT01QTEVYNjQQDhIOCgpDT01QTEVYMTI4EA8S", + "DAoIQkZMT0FUMTYQECKVAQoQVGVuc29yU2hhcGVQcm90bxItCgNkaW0YASAD", + "KAsyIC5vbm54LlRlbnNvclNoYXBlUHJvdG8uRGltZW5zaW9uGlIKCURpbWVu", + "c2lvbhITCglkaW1fdmFsdWUYASABKANIABITCglkaW1fcGFyYW0YAiABKAlI", + "ABISCgpkZW5vdGF0aW9uGAMgASgJQgcKBXZhbHVlIsIECglUeXBlUHJvdG8S", + "LQoLdGVuc29yX3R5cGUYASABKAsyFi5vbm54LlR5cGVQcm90by5UZW5zb3JI", + "ABIxCg1zZXF1ZW5jZV90eXBlGAQgASgLMhgub25ueC5UeXBlUHJvdG8uU2Vx", + "dWVuY2VIABInCghtYXBfdHlwZRgFIAEoCzITLm9ubnguVHlwZVByb3RvLk1h", + "cEgAEi0KC29wYXF1ZV90eXBlGAcgASgLMhYub25ueC5UeXBlUHJvdG8uT3Bh", + "cXVlSAASOgoSc3BhcnNlX3RlbnNvcl90eXBlGAggASgLMhwub25ueC5UeXBl", + "UHJvdG8uU3BhcnNlVGVuc29ySAASEgoKZGVub3RhdGlvbhgGIAEoCRpCCgZU", + "ZW5zb3ISEQoJZWxlbV90eXBlGAEgASgFEiUKBXNoYXBlGAIgASgLMhYub25u", + "eC5UZW5zb3JTaGFwZVByb3RvGi4KCFNlcXVlbmNlEiIKCWVsZW1fdHlwZRgB", + "IAEoCzIPLm9ubnguVHlwZVByb3RvGjwKA01hcBIQCghrZXlfdHlwZRgBIAEo", + "BRIjCgp2YWx1ZV90eXBlGAIgASgLMg8ub25ueC5UeXBlUHJvdG8aJgoGT3Bh", + "cXVlEg4KBmRvbWFpbhgBIAEoCRIMCgRuYW1lGAIgASgJGkgKDFNwYXJzZVRl", + "bnNvchIRCgllbGVtX3R5cGUYASABKAUSJQoFc2hhcGUYAiABKAsyFi5vbm54", + "LlRlbnNvclNoYXBlUHJvdG9CBwoFdmFsdWUiNQoST3BlcmF0b3JTZXRJZFBy", + "b3RvEg4KBmRvbWFpbhgBIAEoCRIPCgd2ZXJzaW9uGAIgASgDKmMKB1ZlcnNp", + "b24SEgoOX1NUQVJUX1ZFUlNJT04QABIZChVJUl9WRVJTSU9OXzIwMTdfMTBf", + "MTAQARIZChVJUl9WRVJTSU9OXzIwMTdfMTBfMzAQAhIOCgpJUl9WRVJTSU9O", + "EANiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Onnx.Version), }, new pbr::GeneratedClrTypeInfo[] { @@ -2116,12 +2114,13 @@ public TensorProto Clone() { /// Field number for the "data_type" field. public const int DataTypeFieldNumber = 2; - private global::Onnx.TensorProto.Types.DataType dataType_ = 0; + private int dataType_; /// /// The data type of the tensor. + /// This field MUST have a valid TensorProto.DataType value /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType DataType { + public int DataType { get { return dataType_; } set { dataType_ = value; @@ -2355,7 +2354,7 @@ public void WriteTo(pb::CodedOutputStream output) { dims_.WriteTo(output, _repeated_dims_codec); if (DataType != 0) { output.WriteRawTag(16); - output.WriteEnum((int) DataType); + output.WriteInt32(DataType); } if (segment_ != null) { output.WriteRawTag(26); @@ -2389,7 +2388,7 @@ public int CalculateSize() { int size = 0; size += dims_.CalculateSize(_repeated_dims_codec); if (DataType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) DataType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(DataType); } if (segment_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Segment); @@ -2462,7 +2461,7 @@ public void MergeFrom(pb::CodedInputStream input) { break; } case 16: { - dataType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + DataType = input.ReadInt32(); break; } case 26: { @@ -3517,13 +3516,14 @@ public Tensor Clone() { /// Field number for the "elem_type" field. public const int ElemTypeFieldNumber = 1; - private global::Onnx.TensorProto.Types.DataType elemType_ = 0; + private int elemType_; /// /// This field MUST NOT have the value of UNDEFINED + /// This field MUST have a valid TensorProto.DataType value /// This field MUST be present for this version of the IR. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType ElemType { + public int ElemType { get { return elemType_; } set { elemType_ = value; @@ -3579,7 +3579,7 @@ public override string ToString() { public void WriteTo(pb::CodedOutputStream output) { if (ElemType != 0) { output.WriteRawTag(8); - output.WriteEnum((int) ElemType); + output.WriteInt32(ElemType); } if (shape_ != null) { output.WriteRawTag(18); @@ -3594,7 +3594,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (ElemType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ElemType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(ElemType); } if (shape_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Shape); @@ -3631,7 +3631,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 8: { - elemType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + ElemType = input.ReadInt32(); break; } case 18: { @@ -3829,13 +3829,14 @@ public Map Clone() { /// Field number for the "key_type" field. public const int KeyTypeFieldNumber = 1; - private global::Onnx.TensorProto.Types.DataType keyType_ = 0; + private int keyType_; /// + /// This field MUST have a valid TensorProto.DataType value /// This field MUST be present for this version of the IR. /// This field MUST refer to an integral type ([U]INT{8|16|32|64}) or STRING /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType KeyType { + public int KeyType { get { return keyType_; } set { keyType_ = value; @@ -3894,7 +3895,7 @@ public override string ToString() { public void WriteTo(pb::CodedOutputStream output) { if (KeyType != 0) { output.WriteRawTag(8); - output.WriteEnum((int) KeyType); + output.WriteInt32(KeyType); } if (valueType_ != null) { output.WriteRawTag(18); @@ -3909,7 +3910,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (KeyType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) KeyType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(KeyType); } if (valueType_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(ValueType); @@ -3946,7 +3947,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 8: { - keyType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + KeyType = input.ReadInt32(); break; } case 18: { @@ -4162,13 +4163,14 @@ public SparseTensor Clone() { /// Field number for the "elem_type" field. public const int ElemTypeFieldNumber = 1; - private global::Onnx.TensorProto.Types.DataType elemType_ = 0; + private int elemType_; /// /// This field MUST NOT have the value of UNDEFINED + /// This field MUST have a valid TensorProto.DataType value /// This field MUST be present for this version of the IR. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType ElemType { + public int ElemType { get { return elemType_; } set { elemType_ = value; @@ -4224,7 +4226,7 @@ public override string ToString() { public void WriteTo(pb::CodedOutputStream output) { if (ElemType != 0) { output.WriteRawTag(8); - output.WriteEnum((int) ElemType); + output.WriteInt32(ElemType); } if (shape_ != null) { output.WriteRawTag(18); @@ -4239,7 +4241,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (ElemType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ElemType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(ElemType); } if (shape_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Shape); @@ -4276,7 +4278,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 8: { - elemType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + ElemType = input.ReadInt32(); break; } case 18: { diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h index dd30683266983..0c5c46eb19af7 100644 --- a/include/onnxruntime/core/framework/kernel_def_builder.h +++ b/include/onnxruntime/core/framework/kernel_def_builder.h @@ -20,13 +20,15 @@ class KernelDefBuilder; typedef std::map MemTypeMap; // note that input/output might be on CPU implicitly when the node is from CPU execution provider -inline bool MemTypeOnCpuExplicitly(const MemTypeMap& mem_type_map, size_t index) { - auto iter = mem_type_map.find(index); - return iter != mem_type_map.end() && (iter->second == OrtMemTypeCPUInput || iter->second == OrtMemTypeCPUOutput); +inline bool MemTypeOnCpuExplicitly(OrtMemType mem_type) { + return mem_type == OrtMemTypeCPUInput || mem_type == OrtMemTypeCPUOutput; } class KernelDef { public: + explicit KernelDef() : default_inputs_mem_type_(OrtMemTypeDefault), default_outputs_mem_type_(OrtMemTypeDefault) { + } + const std::string& OpName() const { return op_name_; } @@ -56,17 +58,20 @@ class KernelDef { return alias_map_; } - const MemTypeMap& InputMemoryType() const { - return input_memory_type_args_; - } - - const MemTypeMap& OutputMemoryType() const { - return output_memory_type_args_; + OrtMemType InputMemoryType(size_t input_index) const { + auto it = input_memory_type_args_.find(input_index); + if (it == input_memory_type_args_.end()) + return default_inputs_mem_type_; + else + return it->second; } - // legacy interface for winml, should not be used in onnxruntime - const MemTypeMap& MemoryType() const { - return output_memory_type_args_; + OrtMemType OutputMemoryType(size_t output_index) const { + auto it = output_memory_type_args_.find(output_index); + if (it == output_memory_type_args_.end()) + return default_outputs_mem_type_; + else + return it->second; } int ExecQueueId() const { @@ -111,6 +116,10 @@ class KernelDef { // execution command queue id, 0 for default queue in execution provider int exec_queue_id_ = 0; + // Default memory type for all inputs + OrtMemType default_inputs_mem_type_; + // Default memory type for all outputs + OrtMemType default_outputs_mem_type_; }; class KernelDefBuilder { @@ -212,6 +221,22 @@ class KernelDefBuilder { return *this; } + /** + Specify the default inputs memory type, if not specified, it is DefaultMemory + */ + KernelDefBuilder& SetDefaultInputsMemoryType(OrtMemType mem_type) { + kernel_def_->default_inputs_mem_type_ = mem_type; + return *this; + } + + /** + Specify the default outputs memory type, if not specified, it is DefaultMemory + */ + KernelDefBuilder& SetDefaultOutputMemoryType(OrtMemType mem_type) { + kernel_def_->default_outputs_mem_type_ = mem_type; + return *this; + } + /** Return the kernel definition, passing ownership of the KernelDef to the caller */ diff --git a/include/onnxruntime/core/framework/onnx_object_cxx.h b/include/onnxruntime/core/framework/onnx_object_cxx.h index f47389ba0cd2c..9677f8bdda980 100644 --- a/include/onnxruntime/core/framework/onnx_object_cxx.h +++ b/include/onnxruntime/core/framework/onnx_object_cxx.h @@ -16,10 +16,10 @@ namespace onnxruntime { template class ObjectBase { private: - static ONNXObject static_cls; + static OrtObject static_cls; protected: - const ONNXObject* const ORT_ATTRIBUTE_UNUSED cls_; + const OrtObject* const ORT_ATTRIBUTE_UNUSED cls_; std::atomic_int ref_count; ObjectBase() : cls_(&static_cls), ref_count(1) { } @@ -39,7 +39,7 @@ class ObjectBase { }; template -ONNXObject ObjectBase::static_cls = {ObjectBase::OrtAddRefImpl, ObjectBase::OrtReleaseImpl}; +OrtObject ObjectBase::static_cls = {ObjectBase::OrtAddRefImpl, ObjectBase::OrtReleaseImpl}; } // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/tensor_shape.h b/include/onnxruntime/core/framework/tensor_shape.h index a012a30d364f8..1007601058640 100644 --- a/include/onnxruntime/core/framework/tensor_shape.h +++ b/include/onnxruntime/core/framework/tensor_shape.h @@ -28,14 +28,18 @@ class TensorShape : private std::vector { public: TensorShape() = default; + TensorShape(const TensorShape& /*other*/) = default; + TensorShape& operator=(const TensorShape& /*other*/) = default; + + TensorShape(TensorShape&& /*other*/) = default; + TensorShape& operator=(TensorShape&& /*other*/) = default; + TensorShape(const int64_t* dimension_sizes, size_t dimension_count); TensorShape(const std::vector& dims); TensorShape(const std::initializer_list& dims); - TensorShape(const TensorShape& /*other*/) = default; - TensorShape(const std::vector& dims, size_t start, size_t end); /** @@ -84,13 +88,16 @@ class TensorShape : private std::vector { /** Return the total number of elements up to the specified dimension. - @param dimension Return size up to this dimension. Value must be >= 0 and < this.Size(). + If the dimension interval is empty (dimension == 0), return 1. + @param dimension Return size up to this dimension. Value must be between 0 and this->NumDimensions(), inclusive. */ int64_t SizeToDimension(size_t dimension) const; /** Return the total number of elements from the specified dimension to the end of the tensor shape. - @param dimension Return size up to this dimension. 0 <= dimension < this.Size(). + If the dimension interval is empty (dimension == this->NumDimensions()), return 1. + @param dimension Return size from this dimension to the end. Value must be between 0 and this->NumDimensions(), + inclusive. */ int64_t SizeFromDimension(size_t dimension) const; @@ -111,7 +118,7 @@ class TensorShape : private std::vector { /** Calculate size between start and end. - Assumes start and end are between 0 and dimensions.size(), inclusive, and that + Assumes start and end are between 0 and this->NumDimensions(), inclusive, and that start < end. */ int64_t SizeHelper(size_t start, size_t end) const; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4c9a8b56402f0..a98a6e0a69aae 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -15,7 +15,7 @@ extern "C" { #endif -// SAL2 staffs +// SAL2 Definitions #ifndef _WIN32 #define _In_ #define _In_opt_ @@ -40,13 +40,15 @@ extern "C" { #endif #define ORT_API_CALL _stdcall #define ORT_MUST_USE_RESULT +#define ORTCHAR_T wchar_t #else #define ORT_EXPORT #define ORT_API_CALL #define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) +#define ORTCHAR_T char #endif -//Any pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. +// Any pointer marked with _In_ or _Out_, cannot be NULL. #ifdef __cplusplus // Windows users should use unicode paths when possible to bypass the MAX_PATH limitation @@ -58,6 +60,46 @@ extern "C" { #define NO_EXCEPTION #endif +// Copied from TensorProto::DataType +// Currently, Ort doesn't support complex64, complex128, bfloat16 types +typedef enum ONNXTensorElementDataType { + ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED = 0, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // maps to c type float + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // maps to c type uint8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // maps to c type int8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // maps to c type uint16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // maps to c type int16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // maps to c type int32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // maps to c type int64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // maps to c++ type std::string + ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, // maps to c type double + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, // maps to c type uint32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, // maps to c type uint64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision +} ONNXTensorElementDataType; + +// Synced with onnx TypeProto oneof +typedef enum ONNXType { + ONNX_TYPE_UNKNOWN, + ONNX_TYPE_TENSOR, + ONNX_TYPE_SEQUENCE, + ONNX_TYPE_MAP, + ONNX_TYPE_OPAQUE, + ONNX_TYPE_SPARSETENSOR, +} ONNXType; + +typedef enum OrtLoggingLevel { + ORT_LOGGING_LEVEL_kVERBOSE = 0, + ORT_LOGGING_LEVEL_kINFO = 1, + ORT_LOGGING_LEVEL_kWARNING = 2, + ORT_LOGGING_LEVEL_kERROR = 3, + ORT_LOGGING_LEVEL_kFATAL = 4 +} OrtLoggingLevel; + typedef enum OrtErrorCode { ORT_OK = 0, ORT_FAIL = 1, @@ -74,200 +116,113 @@ typedef enum OrtErrorCode { ORT_REQUIREMENT_NOT_REGISTERED = 12 } OrtErrorCode; -// ONNXStatus is always returned as a pointer. nullptr indicates success -typedef void ONNXStatus; +// OrtStatus is always returned as a pointer. nullptr indicates success +typedef void OrtStatus; // __VA_ARGS__ on Windows and Linux are different #define ORT_API(RETURN_TYPE, NAME, ...) \ ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION #define ORT_API_STATUS(NAME, ...) \ - ORT_EXPORT ONNXStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT + ORT_EXPORT OrtStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT // Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT #define ORT_API_STATUS_IMPL(NAME, ...) \ - ORT_EXPORT ONNXStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION + ORT_EXPORT OrtStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION -#define DEFINE_RUNTIME_CLASS2(NAME, TYPE) \ - ORT_API(void, Release##NAME, _Frees_ptr_opt_ TYPE* input); +#define ORT_RUNTIME_CLASS2(NAME, TYPE) \ + ORT_API(void, OrtRelease##NAME, _Frees_ptr_opt_ TYPE* input); -#define DEFINE_RUNTIME_CLASS(X) \ - struct X; \ - typedef struct X X; \ - DEFINE_RUNTIME_CLASS2(X, X) +#define ORT_RUNTIME_CLASS(X) \ + struct Ort##X; \ + typedef struct Ort##X Ort##X; \ + ORT_RUNTIME_CLASS2(X, Ort##X) -// ONNXStatus* is pointer to something like this: -// struct ONNXStatus { +// OrtStatus* is pointer to something like this: +// struct OrtStatus { // OrtErrorCode code; // char msg[]; // a null-terminated string, var length // } -DEFINE_RUNTIME_CLASS2(ONNXStatus, void); - -/** - * \param msg A null-terminated string. Its content will be copied into the newly created ONNXStatus - */ -ORT_API(ONNXStatus*, CreateONNXStatus, OrtErrorCode code, _In_ const char* msg) -ORT_ALL_ARGS_NONNULL; - -ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const ONNXStatus* status) -ORT_ALL_ARGS_NONNULL; -/** - * \param status must not be NULL - * \return The error message inside the `status`. Don't free the returned value. - */ -ORT_API(const char*, OrtGetErrorMessage, _In_ const ONNXStatus* status) -ORT_ALL_ARGS_NONNULL; - -// -// Tensor Type and Shapes -// -struct OrtTensorTypeAndShapeInfo; +ORT_RUNTIME_CLASS2(Status, void); -//copied from TensorProto::DataType -//Currently, Ort doesn't support complex64, complex128, bfloat16 types -typedef enum OrtTensorElementDataType { - ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED = 0, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // maps to c type float - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // maps to c type uint8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // maps to c type int8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // maps to c type uint16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // maps to c type int16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // maps to c type int32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // maps to c type int64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // maps to c++ type std::string - ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, // maps to c type double - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, // maps to c type uint32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, // maps to c type uint64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision -} OrtTensorElementDataType; - -//sync with onnx TypeProto oneof -typedef enum OrtType { - ORT_TYPE_UNKNOWN, - ORT_TYPE_TENSOR, - ORT_TYPE_SEQUENCE, - ORT_TYPE_MAP, - ORT_TYPE_OPAQUE, - ORT_TYPE_SPARSETENSOR, -} OrtType; +// The actual types defined have an Ort prefix +ORT_RUNTIME_CLASS(Provider); +ORT_RUNTIME_CLASS(AllocatorInfo); +ORT_RUNTIME_CLASS(Session); +ORT_RUNTIME_CLASS(Value); +ORT_RUNTIME_CLASS(ValueList); struct OrtTypeInfo; - -/** - * Don't free the returned value - */ -ORT_API(const struct OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ struct OrtTypeInfo*); - -/** - * The retured value should be released by calling OrtReleaseObject - */ -ORT_API(struct OrtTensorTypeAndShapeInfo*, OrtCreateTensorTypeAndShapeInfo); - -ORT_API_STATUS(OrtSetTensorElementType, _In_ struct OrtTensorTypeAndShapeInfo*, enum OrtTensorElementDataType type); - -/** - * \param info Created from OrtCreateTensorTypeAndShapeInfo() function - * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. - * \param dim_count length of dim_values - */ -ORT_API_STATUS(OrtSetDims, struct OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - -ORT_API(enum OrtTensorElementDataType, OrtGetTensorElementType, _In_ const struct OrtTensorTypeAndShapeInfo*); -ORT_API(size_t, OrtGetNumOfDimensions, _In_ const struct OrtTensorTypeAndShapeInfo* info); -ORT_API(void, OrtGetDimensions, _In_ const struct OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); - -/** - * How many elements does this tensor have. - * May return a negative value - * e.g. - * [] -> 1 - * [1,3,4] -> 12 - * [2,0,4] -> 0 - * [-1,3,4] -> -1 - * return a negative value if unknown. (That this shape contains a symbolic variable which - * represents an unknown dimension.) - */ -ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const struct OrtTensorTypeAndShapeInfo* info); -struct ONNXValue; - -/** - * \param out Should be freed by OrtReleaseObject after use - */ -ORT_API_STATUS(OrtGetTensorShapeAndType, _In_ const struct ONNXValue* value, - _Out_ struct OrtTensorTypeAndShapeInfo** out); - -/** - * Get the type information of an ONNXValue - * \param value - * \param out The returned value should be freed by OrtReleaseObject after use - */ -ORT_API_STATUS(OrtGetTypeInfo, _In_ const struct ONNXValue* value, struct OrtTypeInfo** out); - -ORT_API(enum OrtType, OrtGetValueType, _In_ const struct ONNXValue* value); - -// -// OrtRunOptions -// +typedef struct OrtTypeInfo OrtTypeInfo; +struct OrtTensorTypeAndShapeInfo; +typedef struct OrtTensorTypeAndShapeInfo OrtTensorTypeAndShapeInfo; struct OrtRunOptions; typedef struct OrtRunOptions OrtRunOptions; +struct OrtSessionOptions; +typedef struct OrtSessionOptions OrtSessionOptions; +struct OrtEnv; +typedef struct OrtEnv OrtEnv; + /** - * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseObject after use + * Every type inherented from OrtObject should be deleted by OrtReleaseObject(...). */ -ORT_API(OrtRunOptions*, OrtCreateRunOptions); +typedef struct OrtObject { + // Returns the new reference count. + uint32_t(ORT_API_CALL* AddRef)(void* this_); + // Returns the new reference count. + uint32_t(ORT_API_CALL* Release)(void* this_); -ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions*, unsigned int); -ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag); +} OrtObject; -ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions*); -ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*); +//inherented from OrtObject +typedef struct OrtAllocatorInterface { + struct OrtObject parent; + void*(ORT_API_CALL* Alloc)(void* this_, size_t size); + void(ORT_API_CALL* Free)(void* this_, void* p); + const struct OrtAllocatorInfo*(ORT_API_CALL* Info)(const void* this_); +} OrtAllocatorInterface; -// set a flag so that any running OrtRunInference* calls that are using this instance of ORtRunOptions -// will exit as soon as possible if the flag is true. -ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions*, _In_ bool value); +typedef OrtAllocatorInterface* OrtAllocator; -DEFINE_RUNTIME_CLASS(OrtProvider); +//Inherented from OrtObject +typedef struct OrtProviderFactoryInterface { + OrtObject parent; + OrtStatus*(ORT_API_CALL* CreateProvider)(void* this_, OrtProvider** out); +} OrtProviderFactoryInterface; -/** - * Just like the IUnknown interface in COM - * Every type inherented from ONNXObject should be deleted by OrtReleaseObject(...). - */ -typedef struct ONNXObject { - ///returns the new reference count. - uint32_t(ORT_API_CALL* AddRef)(void* this_); - ///returns the new reference count. - uint32_t(ORT_API_CALL* Release)(void* this_); - //TODO: implement QueryInterface? -} ONNXObject; +typedef void(ORT_API_CALL* OrtLoggingFunction)( + void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, + const char* message); /** - * This function is a wrapper to "(*(ONNXObject**)ptr)->AddRef(ptr)" - * WARNING: There is NO type checking in this function. - * Before calling this function, caller should make sure current ref count > 0 - * \return the new reference count + * OrtEnv is process-wide. For each process, only one OrtEnv can be created. + * \param out Should be freed by `OrtReleaseObject` after use */ -ORT_API(uint32_t, OrtAddRefToObject, _In_ void* ptr); +ORT_API_STATUS(OrtInitialize, OrtLoggingLevel default_warning_level, _In_ const char* logid, _Out_ OrtEnv** out) +ORT_ALL_ARGS_NONNULL; /** - * - * A wrapper to "(*(ONNXObject**)ptr)->Release(ptr)" - * WARNING: There is NO type checking in this function. - * \param ptr Can be NULL. If it's NULL, this function will return zero. - * \return the new reference count. + * OrtEnv is process-wise. For each process, only one OrtEnv can be created. Don't do it multiple times + * \param out Should be freed by `OrtReleaseObject` after use */ -ORT_API(uint32_t, OrtReleaseObject, _Inout_opt_ void* ptr); +ORT_API_STATUS(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_function, + _In_opt_ void* logger_param, OrtLoggingLevel default_warning_level, + _In_ const char* logid, + _Out_ OrtEnv** out); -//Inherented from ONNXObject -typedef struct OrtProviderFactoryInterface { - ONNXObject parent; - ONNXStatus*(ORT_API_CALL* CreateProvider)(void* this_, OrtProvider** out); -} OrtProviderFactoryInterface; +// TODO: document the path separator convention? '/' vs '\' +// TODO: should specify the access characteristics of model_path. Is this read only during the +// execution of OrtCreateInferenceSession, or does the OrtSession retain a handle to the file/directory +// and continue to access throughout the OrtSession lifetime? +// What sort of access is needed to model_path : read or read/write? +// TODO: allow loading from an in-memory byte-array +ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out); -struct OrtSessionOptions; -typedef struct OrtSessionOptions OrtSessionOptions; +ORT_API_STATUS(OrtRunInference, _Inout_ OrtSession* sess, + _In_ OrtRunOptions* run_options, + _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, + _In_ const char* const* output_names, size_t output_names_len, _Out_ OrtValue** output); /** * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseObject after use @@ -279,11 +234,11 @@ ORT_API(OrtSessionOptions*, OrtCloneSessionOptions, OrtSessionOptions*); ORT_API(void, OrtEnableSequentialExecution, _In_ OrtSessionOptions* options); ORT_API(void, OrtDisableSequentialExecution, _In_ OrtSessionOptions* options); -// enable profiling for this session. +// Enable profiling for this session. ORT_API(void, OrtEnableProfiling, _In_ OrtSessionOptions* options, _In_ const char* profile_file_prefix); ORT_API(void, OrtDisableProfiling, _In_ OrtSessionOptions* options); -// enable the memory pattern optimization. +// Enable the memory pattern optimization. // The idea is if the input shapes are the same, we could trace the internal memory allocation // and generate a memory pattern for future request. So next time we could just do one allocation // with a big chunk for all the internal memory allocation. @@ -314,185 +269,216 @@ ORT_API(void, OrtSessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* ORT_API(void, OrtAddCustomOp, _In_ OrtSessionOptions* options, const char* custom_op_path); -typedef enum OrtAllocatorType { - OrtDeviceAllocator = 0, - OrtArenaAllocator = 1 -} OrtAllocatorType; +ORT_API_STATUS(OrtInferenceSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); +ORT_API_STATUS(OrtInferenceSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); /** - memory types for allocator, exec provider specific types should be extended in each provider -*/ -typedef enum OrtMemType { - OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider - OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeDefault = 0, // the default allocator for execution provider -} OrtMemType; - -DEFINE_RUNTIME_CLASS(OrtAllocatorInfo); - -ORT_API_STATUS(OrtCreateAllocatorInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out); - -/** - * Test if two allocation info are equal - * \return 0, equal. zero, not equal - */ -ORT_API(int, OrtCompareAllocatorInfo, _In_ const OrtAllocatorInfo* info1, _In_ const OrtAllocatorInfo* info2) -ORT_ALL_ARGS_NONNULL; -/** - * Do not free the returned value + * \param out should be freed by OrtReleaseObject after use */ -ORT_API(const char*, OrtAllocatorInfoGetName, _In_ OrtAllocatorInfo* ptr); -ORT_API(int, OrtAllocatorInfoGetId, _In_ OrtAllocatorInfo* ptr); -ORT_API(OrtMemType, OrtAllocatorInfoGetMemType, _In_ OrtAllocatorInfo* ptr); -ORT_API(OrtAllocatorType, OrtAllocatorInfoGetType, _In_ OrtAllocatorInfo* ptr); - -//inherented from ONNXObject -typedef struct OrtAllocatorInterface { - struct ONNXObject parent; - void*(ORT_API_CALL* Alloc)(void* this_, size_t size); - void(ORT_API_CALL* Free)(void* this_, void* p); - const struct OrtAllocatorInfo*(ORT_API_CALL* Info)(const void* this_); -} OrtAllocatorInterface; +ORT_API_STATUS(OrtInferenceSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); -typedef OrtAllocatorInterface* OrtAllocator; - -ORT_API(void*, OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size); -ORT_API(void, OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); -ORT_API(const struct OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr); - -struct OrtEnv; -typedef struct OrtEnv OrtEnv; - -typedef enum OrtLoggingLevel { - ORT_LOGGING_LEVEL_kVERBOSE = 0, - ORT_LOGGING_LEVEL_kINFO = 1, - ORT_LOGGING_LEVEL_kWARNING = 2, - ORT_LOGGING_LEVEL_kERROR = 3, - ORT_LOGGING_LEVEL_kFATAL = 4 -} OrtLoggingLevel; - -typedef void(ORT_API_CALL* OrtLoggingFunction)( - void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, - const char* message); /** - * OrtEnv is process-wise. For each process, only one OrtEnv can be created. Don't do it multiple times - * \param out Should be freed by `OrtReleaseObject` after use + * \param out should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtInitialize, OrtLoggingLevel default_warning_level, _In_ const char* logid, _Out_ OrtEnv** out) -ORT_ALL_ARGS_NONNULL; +ORT_API_STATUS(OrtInferenceSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); + +ORT_API_STATUS(OrtInferenceSessionGetInputName, _In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, _Out_ char** value); +ORT_API_STATUS(OrtInferenceSessionGetOutputName, _In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, _Out_ char** value); /** - * OrtEnv is process-wise. For each process, only one OrtEnv can be created. Don't do it multiple times - * \param out Should be freed by `OrtReleaseObject` after use + * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_function, - _In_opt_ void* logger_param, OrtLoggingLevel default_warning_level, - _In_ const char* logid, - _Out_ OrtEnv** out); - -DEFINE_RUNTIME_CLASS(ONNXSession); +ORT_API(OrtRunOptions*, OrtCreateRunOptions); -//TODO: document the path separator convention? '/' vs '\' -//TODO: should specify the access characteristics of model_path. Is this read only during the -//execution of OrtCreateInferenceSession, or does the ONNXSession retain a handle to the file/directory -//and continue to access throughout the ONNXSession lifetime? -// What sort of access is needed to model_path : read or read/write? -//TODO: allow loading from an in-memory byte-array -#ifdef _WIN32 -ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const wchar_t* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out); -#else -ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const char* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out); -#endif +ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions*, unsigned int); +ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag); -DEFINE_RUNTIME_CLASS(ONNXValue); +ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions*); +ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*); -///Call OrtReleaseObject to release the returned value -ORT_API_STATUS(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out); +// set a flag so that any running OrtRunInference* calls that are using this instance of ORtRunOptions +// will exit as soon as possible if the flag is true. +ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions*, _In_ bool value); /** - * Create a tensor from an allocator. ReleaseONNXValue will also release the buffer inside the output value + * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value * \param out will keep a reference to the allocator, without reference counting(will be fixed). Should be freed by - * calling ReleaseONNXValue + * calling OrtReleaseValue * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx */ -ORT_API_STATUS(OrtCreateTensorAsONNXValue, _Inout_ OrtAllocator* allocator, - _In_ const size_t* shape, size_t shape_len, OrtTensorElementDataType type, - _Out_ ONNXValue** out); +ORT_API_STATUS(OrtCreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const size_t* shape, size_t shape_len, ONNXTensorElementDataType type, + _Out_ OrtValue** out); /** * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. - * p_data is owned by caller. ReleaseONNXValue won't release p_data. - * \param out Should be freed by calling ReleaseONNXValue + * p_data is owned by caller. OrtReleaseValue won't release p_data. + * \param out Should be freed by calling OrtReleaseValue */ -ORT_API_STATUS(OrtCreateTensorWithDataAsONNXValue, _In_ const OrtAllocatorInfo* info, +ORT_API_STATUS(OrtCreateTensorWithDataAsOrtValue, _In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, - OrtTensorElementDataType type, _Out_ ONNXValue** out); + ONNXTensorElementDataType type, _Out_ OrtValue** out); -/// This function doesn't work with string tensor -/// this is a no-copy method whose pointer is only valid until the backing ONNXValue is free'd. -ORT_API_STATUS(OrtGetTensorMutableData, _Inout_ ONNXValue* value, _Out_ void** out); +// This function doesn't work with string tensor +// this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd. +ORT_API_STATUS(OrtGetTensorMutableData, _Inout_ OrtValue* value, _Out_ void** out); /** - * Test if an ONNXValue is a tensor - * \return zero, false. non-zero true + * Test if an OrtValue is a tensor + * \return zero if false. non-zero if true */ -ORT_API(int, OrtIsTensor, _In_ const ONNXValue* value); +ORT_API(int, OrtIsTensor, _In_ const OrtValue* value); /** - * \param value A tensor created from OrtCreateTensor*** function. + * \param value A tensor created from OrtCreateTensor... function. * \param s each A string array. Each string in this array must be null terminated. * \param s_len length of s */ -ORT_API_STATUS(OrtFillStringTensor, _In_ ONNXValue* value, _In_ const char* const* s, size_t s_len); +ORT_API_STATUS(OrtFillStringTensor, _In_ OrtValue* value, _In_ const char* const* s, size_t s_len); /** - * \param value A tensor created from OrtCreateTensor*** function. + * \param value A tensor created from OrtCreateTensor... function. * \param len total data length, not including the trailing '\0' chars. */ -ORT_API_STATUS(OrtGetStringTensorDataLength, _In_ const ONNXValue* value, _Out_ size_t* len); +ORT_API_STATUS(OrtGetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); /** * \param s string contents. Each string is NOT null-terminated. - * \param value A tensor created from OrtCreateTensor*** function. + * \param value A tensor created from OrtCreateTensor... function. * \param s_len total data length, get it from OrtGetStringTensorDataLength */ -ORT_API_STATUS(OrtGetStringTensorContent, _In_ const ONNXValue* value, _Out_ void* s, size_t s_len, +ORT_API_STATUS(OrtGetStringTensorContent, _In_ const OrtValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len); -DEFINE_RUNTIME_CLASS(ONNXValueList); +ORT_API_STATUS(OrtTensorProtoToOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const void* input, int input_len, _Out_ OrtValue** out); -ORT_API_STATUS(OrtRunInference, _Inout_ ONNXSession* sess, - _In_ OrtRunOptions* run_options, - _In_ const char* const* input_names, _In_ const ONNXValue* const* input, size_t input_len, - _In_ const char* const* output_names, size_t output_names_len, _Out_ ONNXValue** output); +/** + * Don't free the returned value + */ +ORT_API(const OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ OrtTypeInfo*); + +/** + * The retured value should be released by calling OrtReleaseObject + */ +ORT_API(OrtTensorTypeAndShapeInfo*, OrtCreateTensorTypeAndShapeInfo); -ORT_API_STATUS(OrtInferenceSessionGetInputCount, _In_ const ONNXSession* sess, _Out_ size_t* out); -ORT_API_STATUS(OrtInferenceSessionGetOutputCount, _In_ const ONNXSession* sess, _Out_ size_t* out); +ORT_API_STATUS(OrtSetTensorElementType, _In_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type); /** - * \param out should be freed by OrtReleaseObject after use + * \param info Created from OrtCreateTensorTypeAndShapeInfo() function + * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. + * \param dim_count length of dim_values */ -ORT_API_STATUS(OrtInferenceSessionGetInputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out); +ORT_API_STATUS(OrtSetDims, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); + +ORT_API(enum ONNXTensorElementDataType, OrtGetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*); +ORT_API(size_t, OrtGetNumOfDimensions, _In_ const OrtTensorTypeAndShapeInfo* info); +ORT_API(void, OrtGetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); /** - * \param out should be freed by OrtReleaseObject after use + * How many elements does this tensor have. + * May return a negative value + * e.g. + * [] -> 1 + * [1,3,4] -> 12 + * [2,0,4] -> 0 + * [-1,3,4] -> -1 + * return a negative value if unknown. (That this shape contains a symbolic variable which + * represents an unknown dimension.) */ -ORT_API_STATUS(OrtInferenceSessionGetOutputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out); +ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info); -ORT_API_STATUS(OrtInferenceSessionGetInputName, _In_ const ONNXSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, _Out_ char** value); -ORT_API_STATUS(OrtInferenceSessionGetOutputName, _In_ const ONNXSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, _Out_ char** value); +/** + * \param out Should be freed by OrtReleaseObject after use + */ +ORT_API_STATUS(OrtGetTensorShapeAndType, _In_ const OrtValue* value, _Out_ OrtTensorTypeAndShapeInfo** out); + +/** + * Get the type information of an OrtValue + * \param value + * \param out The returned value should be freed by OrtReleaseObject after use + */ +ORT_API_STATUS(OrtGetTypeInfo, _In_ const OrtValue* value, OrtTypeInfo** out); + +ORT_API(enum ONNXType, OrtGetValueType, _In_ const OrtValue* value); -ORT_API_STATUS(OrtTensorProtoToONNXValue, _Inout_ OrtAllocator* allocator, - _In_ const void* input, int input_len, _Out_ ONNXValue** out); +/** + * This function is a wrapper to "(*(OrtObject**)ptr)->AddRef(ptr)" + * WARNING: There is NO type checking in this function. + * Before calling this function, caller should make sure current ref count > 0 + * \return the new reference count + */ +ORT_API(uint32_t, OrtAddRefToObject, _In_ void* ptr); + +/** + * + * A wrapper to "(*(OrtObject**)ptr)->Release(ptr)" + * WARNING: There is NO type checking in this function. + * \param ptr Can be NULL. If it's NULL, this function will return zero. + * \return the new reference count. + */ +ORT_API(uint32_t, OrtReleaseObject, _Inout_opt_ void* ptr); + +typedef enum OrtAllocatorType { + OrtDeviceAllocator = 0, + OrtArenaAllocator = 1 +} OrtAllocatorType; + +/** + memory types for allocator, exec provider specific types should be extended in each provider +*/ +typedef enum OrtMemType { + OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider + OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED + OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED + OrtMemTypeDefault = 0, // the default allocator for execution provider +} OrtMemType; + +ORT_API_STATUS(OrtCreateAllocatorInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out); + +/** + * Test if two allocation info are equal + * \return 0, equal. zero, not equal + */ +ORT_API(int, OrtCompareAllocatorInfo, _In_ const OrtAllocatorInfo* info1, _In_ const OrtAllocatorInfo* info2) +ORT_ALL_ARGS_NONNULL; + +/** + * Do not free the returned value + */ +ORT_API(const char*, OrtAllocatorInfoGetName, _In_ OrtAllocatorInfo* ptr); +ORT_API(int, OrtAllocatorInfoGetId, _In_ OrtAllocatorInfo* ptr); +ORT_API(OrtMemType, OrtAllocatorInfoGetMemType, _In_ OrtAllocatorInfo* ptr); +ORT_API(OrtAllocatorType, OrtAllocatorInfoGetType, _In_ OrtAllocatorInfo* ptr); + +ORT_API(void*, OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size); +ORT_API(void, OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); +ORT_API(const OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr); + +// Call OrtReleaseObject to release the returned value +ORT_API_STATUS(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out); + +/** + * \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus + */ +ORT_API(OrtStatus*, OrtCreateStatus, OrtErrorCode code, _In_ const char* msg) +ORT_ALL_ARGS_NONNULL; + +ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const OrtStatus* status) +ORT_ALL_ARGS_NONNULL; +/** + * \param status must not be NULL + * \return The error message inside the `status`. Don't free the returned value. + */ +ORT_API(const char*, OrtGetErrorMessage, _In_ const OrtStatus* status) +ORT_ALL_ARGS_NONNULL; /** * Deprecated. Please use OrtReleaseObject */ -ORT_API(void, ReleaseONNXEnv, OrtEnv* env); +ORT_API(void, OrtReleaseEnv, OrtEnv* env); #ifdef __cplusplus } diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index d9503688ee125..72ea38c995dbf 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -11,10 +11,10 @@ //TODO: encode error code in the message? #define ORT_THROW_ON_ERROR(expr) \ do { \ - ONNXStatus* onnx_status = (expr); \ + OrtStatus* onnx_status = (expr); \ if (onnx_status != nullptr) { \ std::string ort_error_message = OrtGetErrorMessage(onnx_status); \ - ReleaseONNXStatus(onnx_status); \ + OrtReleaseStatus(onnx_status); \ throw std::runtime_error(ort_error_message); \ } \ } while (0); @@ -29,7 +29,7 @@ template <> \ struct default_delete { \ void operator()(Ort##TYPE_NAME* ptr) { \ - (*reinterpret_cast(ptr))->Release(ptr); \ + (*reinterpret_cast(ptr))->Release(ptr); \ } \ }; \ } @@ -89,14 +89,14 @@ class SessionOptionsWrapper { return SessionOptionsWrapper(env_, p); } #ifdef _WIN32 - ONNXSession* OrtCreateInferenceSession(_In_ const wchar_t* model_path) { - ONNXSession* ret; + OrtSession* OrtCreateInferenceSession(_In_ const wchar_t* model_path) { + OrtSession* ret; ORT_THROW_ON_ERROR(::OrtCreateInferenceSession(env_, model_path, value.get(), &ret)); return ret; } #else - ONNXSession* OrtCreateInferenceSession(_In_ const char* model_path) { - ONNXSession* ret; + OrtSession* OrtCreateInferenceSession(_In_ const char* model_path) { + OrtSession* ret; ORT_THROW_ON_ERROR(::OrtCreateInferenceSession(env_, model_path, value.get(), &ret)); return ret; } @@ -105,15 +105,15 @@ class SessionOptionsWrapper { OrtAddCustomOp(value.get(), custom_op_path); } }; -inline ONNXValue* OrtCreateTensorAsONNXValue(_Inout_ OrtAllocator* env, const std::vector& shape, OrtTensorElementDataType type) { - ONNXValue* ret; - ORT_THROW_ON_ERROR(::OrtCreateTensorAsONNXValue(env, shape.data(), shape.size(), type, &ret)); +inline OrtValue* OrtCreateTensorAsOrtValue(_Inout_ OrtAllocator* env, const std::vector& shape, ONNXTensorElementDataType type) { + OrtValue* ret; + ORT_THROW_ON_ERROR(::OrtCreateTensorAsOrtValue(env, shape.data(), shape.size(), type, &ret)); return ret; } -inline ONNXValue* OrtCreateTensorWithDataAsONNXValue(_In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, OrtTensorElementDataType type) { - ONNXValue* ret; - ORT_THROW_ON_ERROR(::OrtCreateTensorWithDataAsONNXValue(info, p_data, p_data_len, shape.data(), shape.size(), type, &ret)); +inline OrtValue* OrtCreateTensorWithDataAsOrtValue(_In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, ONNXTensorElementDataType type) { + OrtValue* ret; + ORT_THROW_ON_ERROR(::OrtCreateTensorWithDataAsOrtValue(info, p_data, p_data_len, shape.data(), shape.size(), type, &ret)); return ret; } diff --git a/onnxruntime/contrib_ops/contrib_kernels.cc b/onnxruntime/contrib_ops/contrib_kernels.cc index 11b7e653f3058..a8f8876e771a2 100644 --- a/onnxruntime/contrib_ops/contrib_kernels.cc +++ b/onnxruntime/contrib_ops/contrib_kernels.cc @@ -17,6 +17,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, string, StringNormalizer); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, NonMaxSuppression); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Range); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3); void RegisterContribKernels(std::function fn) { @@ -33,6 +34,7 @@ void RegisterContribKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); fn(BuildKernel()); } } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/gather_nd.cc b/onnxruntime/contrib_ops/cpu/gather_nd.cc new file mode 100644 index 0000000000000..f13794aa6b735 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/gather_nd.cc @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "contrib_ops/cpu/gather_nd.h" + +namespace onnxruntime { +namespace contrib { + +ONNX_OPERATOR_KERNEL_EX( + GatherND, + kMSDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .TypeConstraint("Tind", {DataTypeImpl::GetTensorType(),DataTypeImpl::GetTensorType()}), + GatherND); + +template +Status GatherNDBase::PrepareForCompute(OpKernelContext* context, Prepare& p) const { + + auto input_tensor = context->Input(0); + auto indice_tensor = context->Input(1); + ORT_ENFORCE(input_tensor != nullptr); + ORT_ENFORCE(indice_tensor != nullptr); + + auto input_shape = input_tensor->Shape(); + auto indice_shape = indice_tensor->Shape(); + if (indice_shape.NumDimensions() == 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "indices tensor must has rank larger than 0"); + } + + auto last_indice_dimension = indice_shape[indice_shape.NumDimensions() - 1]; + if (last_indice_dimension > static_cast(input_shape.NumDimensions())) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "last dimension of indices must not be larger than rank of input tensor"); + } + + std::vector shape(indice_shape.GetDims().begin(), + indice_shape.GetDims().end() - 1); + shape.insert(shape.end(), + input_shape.GetDims().begin() + last_indice_dimension, + input_shape.GetDims().end()); + auto output_tensor = context->Output(0,TensorShape(shape)); + std::vector element_counts(last_indice_dimension, 0LL); // Number of elements for each input dimension + +#pragma omp parallel for + for (int64_t i = 0; i < last_indice_dimension; ++i) { + element_counts[i] = input_shape.SizeFromDimension(i + 1); + } + + int64_t err_indice = 0; + p.element_bytes = input_tensor->DataType()->Size(); + p.element_to_copy = input_shape.SizeFromDimension(last_indice_dimension); + p.bytes_to_copy = p.element_bytes * p.element_to_copy; + auto indice_offset = static_cast(context->Input(1)->DataRaw()); + auto offset_count = indice_shape.Size() / last_indice_dimension; // Times to copy + p.element_offsets.assign(offset_count, 0LL); + + if (input_tensor->DataType() == DataTypeImpl::GetType()) { + p.input_str_base = static_cast(input_tensor->DataRaw()); + p.output_str_base = static_cast(output_tensor->MutableDataRaw()); + } else { + p.input_base = static_cast(context->Input(0)->DataRaw()); + p.output_base = static_cast(output_tensor->MutableDataRaw()); + } + +#pragma omp parallel for + for (int64_t i = 0; i < offset_count; ++i) { + for (int64_t j = 0; j < last_indice_dimension; ++j) { + auto indice = *(indice_offset + i * last_indice_dimension + j); + if (indice < 0 || indice >= input_shape[j]) { + err_indice = indice; + } + p.element_offsets[i] += indice * element_counts[j]; + } + } + return err_indice == 0 ? Status::OK() : + ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "invalid indice found, indice = ", err_indice); +} + +template Status GatherNDBase::PrepareForCompute(OpKernelContext*, Prepare&) const; +template Status GatherNDBase::PrepareForCompute(OpKernelContext*, Prepare&) const; + +Status GatherND::Compute(OpKernelContext* context) const { + Prepare p; + ORT_RETURN_IF_ERROR(context->Input(1)->DataType() == DataTypeImpl::GetType() ? + PrepareForCompute(context, p) : PrepareForCompute(context, p)); + return nullptr == p.input_str_base ? GatherNumber(p) : GatherString(p); +} + +Status GatherND::GatherNumber(const Prepare& p) const { +#pragma omp parallel for + for (int64_t i = 0; i < static_cast(p.element_offsets.size()); ++i) { + memcpy(p.output_base + i * p.bytes_to_copy, + p.input_base + p.element_offsets[i] * p.element_bytes, + p.bytes_to_copy); + } + return Status::OK(); +} + +Status GatherND::GatherString(const Prepare& p) const { +#pragma omp parallel for + for (int64_t i = 0; i < static_cast(p.element_offsets.size()); ++i) { + for (int64_t j = 0; j < static_cast(p.element_to_copy); ++j) { + p.output_str_base[i * p.element_to_copy + j] = p.input_str_base[p.element_offsets[i] + j]; + } + } + return Status::OK(); +} + +} +} diff --git a/onnxruntime/contrib_ops/cpu/gather_nd.h b/onnxruntime/contrib_ops/cpu/gather_nd.h new file mode 100644 index 0000000000000..6d256e07fd8ab --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/gather_nd.h @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { + +class GatherNDBase +{ +protected: + struct Prepare { + const uint8_t* input_base; + const std::string* input_str_base; + uint8_t* output_base; + std::string* output_str_base; + uint64_t bytes_to_copy; + uint64_t element_bytes; + uint64_t element_to_copy; + std::vector element_offsets; + + Prepare(): input_base (nullptr), + input_str_base (nullptr), + output_base (nullptr), + output_str_base (nullptr), + bytes_to_copy (0), + element_bytes (0), + element_to_copy (0), + element_offsets (0) {} + }; // struct Prepare + + template + Status PrepareForCompute(OpKernelContext* context, Prepare& p) const; +}; // class GatherNDBase + +class GatherND final : public OpKernel, protected GatherNDBase { +public: + explicit GatherND(const OpKernelInfo& info) : OpKernel(info) {} + Status Compute(OpKernelContext* context) const override; +private: + Status GatherNumber(const Prepare& p) const; + Status GatherString(const Prepare& p) const; +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/string_normalizer.cc b/onnxruntime/contrib_ops/cpu/string_normalizer.cc index 2b76a7bbb486b..a20eaade10d3b 100644 --- a/onnxruntime/contrib_ops/cpu/string_normalizer.cc +++ b/onnxruntime/contrib_ops/cpu/string_normalizer.cc @@ -42,7 +42,7 @@ class Locale { loc_ = _create_locale(LC_CTYPE, name.c_str()); if (loc_ == nullptr) { ORT_THROW("Failed to construct locale with name:", - name, ":", ":Please, install necessary language-pack-XX and configure locales"); + name, ":", ":Please, install necessary language-pack-XX and configure locales"); } } @@ -78,7 +78,7 @@ class Locale { explicit Locale(const std::string& name) try : loc_(name) { } catch (const std::runtime_error& e) { ORT_THROW("Failed to construct locale with name:", - name, ":", e.what(), ":Please, install necessary language-pack-XX and configure locales"); + name, ":", e.what(), ":Please, install necessary language-pack-XX and configure locales"); } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Locale); @@ -118,9 +118,8 @@ Status CopyCaseAction(ForwardIter first, ForwardIter end, OpKernelContext* ctx, if (C == 0) { output_dims.push_back(1); TensorShape output_shape(output_dims); - auto output_ten = ctx->Output(0, output_shape); - auto output_default = output_ten->template MutableData(); - new (output_default) std::string(); + // This will create one empty string + ctx->Output(0, output_shape); return Status::OK(); } @@ -141,11 +140,11 @@ Status CopyCaseAction(ForwardIter first, ForwardIter end, OpKernelContext* ctx, } // In place transform loc.ChangeCase(caseaction, wstr); - new (output_data + output_idx) std::string(converter.to_bytes(wstr)); + *(output_data + output_idx) = converter.to_bytes(wstr); } else { assert(caseaction == StringNormalizer::NONE); // Simple copy or move if the iterator points to a non-const string - new (output_data + output_idx) std::string(std::move(s)); + *(output_data + output_idx) = std::move(s); } ++output_idx; ++first; diff --git a/onnxruntime/contrib_ops/cpu/tokenizer.cc b/onnxruntime/contrib_ops/cpu/tokenizer.cc index 90aaeb9ab8e7e..4c31ec20e596d 100644 --- a/onnxruntime/contrib_ops/cpu/tokenizer.cc +++ b/onnxruntime/contrib_ops/cpu/tokenizer.cc @@ -218,7 +218,7 @@ Tokenizer::Tokenizer(const OpKernelInfo& info) : OpKernel(info) { separators[0].empty()); ORT_ENFORCE(!char_tokenezation_ || mincharnum_ < 2, - "mincharnum is too big for char level tokenezation"); + "mincharnum is too big for char level tokenezation"); // Create TST and insert separators if (!char_tokenezation_) { @@ -284,7 +284,7 @@ Status Tokenizer::CharTokenize(OpKernelContext* ctx, size_t N, size_t C, while (curr_input != last) { const auto& s = *curr_input; if (mark_) { - new (output_data + output_index) std::string(&start_text, 1); + (output_data + output_index)->assign(&start_text, 1); ++output_index; } size_t tokens = 0; @@ -295,20 +295,20 @@ Status Tokenizer::CharTokenize(OpKernelContext* ctx, size_t N, size_t C, assert(result); (void)result; assert(token_idx + tlen <= str_len); - new (output_data + output_index) std::string(s.substr(token_idx, tlen)); + *(output_data + output_index) = s.substr(token_idx, tlen); ++output_index; token_idx += tlen; ++tokens; } if (mark_) { - new (output_data + output_index) std::string(&end_text, 1); + (output_data + output_index)->assign(&end_text, 1); ++output_index; } // Padding strings assert(tokens + (mark_ * 2) <= max_tokens); const size_t pads = max_tokens - (mark_ * 2) - tokens; for (size_t p = 0; p < pads; ++p) { - new (output_data + output_index) std::string(pad_value_); + *(output_data + output_index) = pad_value_; ++output_index; } ++curr_input; @@ -422,21 +422,21 @@ Status Tokenizer::SeparatorTokenize(OpKernelContext* ctx, size_t c_idx = output_index; #endif if (mark_) { - new (output_data + output_index) std::string(&start_text, 1); + (output_data + output_index)->assign(&start_text, 1); ++output_index; } // Output tokens for this row for (auto& token : row) { - new (output_data + output_index) std::string(converter.to_bytes(token)); + *(output_data + output_index) = converter.to_bytes(token); ++output_index; } if (mark_) { - new (output_data + output_index) std::string(&end_text, 1); + (output_data + output_index)->assign(&end_text, 1); ++output_index; } const size_t pads = max_tokens - (mark_ * 2) - row.size(); for (size_t p = 0; p < pads; ++p) { - new (output_data + output_index) std::string(pad_value_); + *(output_data + output_index) = pad_value_; ++output_index; } #ifdef _DEBUG diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc index dab771b9bc903..c0946b88f48e9 100644 --- a/onnxruntime/core/common/cpuid_info.cc +++ b/onnxruntime/core/common/cpuid_info.cc @@ -56,6 +56,7 @@ CPUIDInfo::CPUIDInfo() noexcept { int value = XGETBV(); bool has_avx = (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK); bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK; + has_f16c_ = has_avx && (data[2] & (1 << 29)) && (data[3] & (1 << 26)); if (num_IDs >= 7) { GetCPUID(7, data); diff --git a/onnxruntime/core/common/cpuid_info.h b/onnxruntime/core/common/cpuid_info.h index 2a1b89d9e70fd..de6c57431a13b 100644 --- a/onnxruntime/core/common/cpuid_info.h +++ b/onnxruntime/core/common/cpuid_info.h @@ -14,11 +14,13 @@ class CPUIDInfo { bool HasAVX2() const { return has_avx2_; } bool HasAVX512f() const { return has_avx512f_; } + bool HasF16C() const { return has_f16c_; } private: CPUIDInfo() noexcept; bool has_avx2_{false}; bool has_avx512f_{false}; + bool has_f16c_{false}; }; } diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index c13d60b71fab7..9bc25e9303fda 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -380,7 +380,6 @@ class PlannerImpl { ORT_ENFORCE(exec_provider); auto& default_allocator_info = exec_provider->GetAllocator(0, OrtMemTypeDefault)->Info(); - auto& mem_type_allocated_args = p_kernelDef->OutputMemoryType(); auto& outputs = pnode->OutputDefs(); auto num_outputs = outputs.size(); @@ -389,14 +388,15 @@ class PlannerImpl { if (node_output->Exists()) { MLValueIndex index = Index(node_output->Name()); ProcessDef(index, node_output); + ++UseCount(index); if (strcmp(default_allocator_info.name, CPU) != 0) { // By default, outputs of this node are allocated on the default device allocator, // except for outputs marked for allocation in MemoryType: - auto memory_type_iter = mem_type_allocated_args.find(i); - if (memory_type_iter == mem_type_allocated_args.end()) { + auto memory_type = p_kernelDef->OutputMemoryType(i); + if (memory_type == OrtMemTypeDefault) { AllocPlan(index).location = default_allocator_info; } else { - AllocPlan(index).location = exec_provider->GetAllocator(0, memory_type_iter->second)->Info(); + AllocPlan(index).location = exec_provider->GetAllocator(0, memory_type)->Info(); } } } @@ -437,7 +437,7 @@ class PlannerImpl { thisplan.alloc_kind = AllocKind::kAllocateStatically; auto p_opkernelDef = utils::GetKernelDef(kernel_registry_, node); - if (MemTypeOnCpuExplicitly(p_opkernelDef->InputMemoryType(), index)) + if (MemTypeOnCpuExplicitly(p_opkernelDef->InputMemoryType(index))) // weights are not output from any node, so it's OK to put its location on CPU provider thisplan.location = execution_providers_.Get(onnxruntime::kCpuExecutionProvider)->GetAllocator(0, OrtMemTypeDefault)->Info(); else @@ -528,7 +528,7 @@ class PlannerImpl { if (node_output->Exists()) { auto& sym = node_output->Name(); auto original = Buffer(Index(sym)); - if (0 == UseCount(original)) + if (0 == --UseCount(original)) freelist_.push_front(FreeBufferInfo(original, program_counter)); } } diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc index ca9d5e158e183..26eaac40c7e64 100644 --- a/onnxruntime/core/framework/allocator.cc +++ b/onnxruntime/core/framework/allocator.cc @@ -36,7 +36,7 @@ ORT_API_STATUS_IMPL(OrtCreateAllocatorInfo, const char* name1, OrtAllocatorType return nullptr; } -ORT_API(void, ReleaseOrtAllocatorInfo, OrtAllocatorInfo* p) { +ORT_API(void, OrtReleaseAllocatorInfo, OrtAllocatorInfo* p) { delete p; } diff --git a/onnxruntime/core/framework/error_code.cc b/onnxruntime/core/framework/error_code.cc index 8513905034f43..87cff0ddbfffe 100644 --- a/onnxruntime/core/framework/error_code.cc +++ b/onnxruntime/core/framework/error_code.cc @@ -7,7 +7,7 @@ #include using onnxruntime::common::Status; -ORT_API(ONNXStatus*, CreateONNXStatus, OrtErrorCode code, const char* msg) { +ORT_API(OrtStatus*, OrtCreateStatus, OrtErrorCode code, const char* msg) { assert(!(code == 0 && msg != nullptr)); size_t clen = strlen(msg); size_t len = clen + 1 + sizeof(int); @@ -21,7 +21,7 @@ ORT_API(ONNXStatus*, CreateONNXStatus, OrtErrorCode code, const char* msg) { return ret; } namespace onnxruntime { -ONNXStatus* ToONNXStatus(const Status& st) { +OrtStatus* ToOrtStatus(const Status& st) { if (st.IsOK()) return nullptr; size_t clen = st.ErrorMessage().length(); @@ -36,10 +36,10 @@ ONNXStatus* ToONNXStatus(const Status& st) { return ret; } } // namespace onnxruntime -ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const ONNXStatus* status) { - return *reinterpret_cast(const_cast(status)); +ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const OrtStatus* status) { + return *reinterpret_cast(const_cast(status)); } -ORT_API(const char*, OrtGetErrorMessage, _In_ const ONNXStatus* status) { +ORT_API(const char*, OrtGetErrorMessage, _In_ const OrtStatus* status) { return reinterpret_cast(status) + sizeof(int); } diff --git a/onnxruntime/core/framework/error_code_helper.h b/onnxruntime/core/framework/error_code_helper.h index 9b75e7a138e50..90c02f5f7b3e4 100644 --- a/onnxruntime/core/framework/error_code_helper.h +++ b/onnxruntime/core/framework/error_code_helper.h @@ -6,5 +6,5 @@ #include "core/common/status.h" namespace onnxruntime { -ONNXStatus* ToONNXStatus(const onnxruntime::common::Status& st); +OrtStatus* ToOrtStatus(const onnxruntime::common::Status& st); }; diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index c5323c4e0a0ed..9e3c122f21c58 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -89,7 +89,7 @@ Status ExecutionFrame::AllocateMLValueTensorSelfOwnBufferHelper(int mlvalue_inde if (len < 0) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Tensor shape cannot contain any negative value"); } - if (!IAllocator::CalcMemSizeForArray(len, element_type->Size(), &size)) { + if (!IAllocator::CalcMemSizeForArrayWithAlignment<64>(len, element_type->Size(), &size)) { return Status(ONNXRUNTIME, FAIL, "size overflow"); } } @@ -284,7 +284,7 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(int mlvalue_index, ORT_RETURN_IF_ERROR(AllocateMLValueTensorSelfOwnBuffer(mlvalue_index, ml_data_type, alloc_info, - parameters.tensor_shape, + parameters.GetTensorShape(), per_alloc_plan.create_fence_if_async)); break; } @@ -294,7 +294,7 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(int mlvalue_index, reuse_mlvalue_index, ml_data_type, alloc_info, - parameters.tensor_shape, + parameters.GetTensorShape(), per_alloc_plan.create_fence_if_async)); break; } @@ -363,6 +363,15 @@ void ExecutionFrame::Init(const onnxruntime::GraphViewer& graph, } // 5. set node args + std::size_t total_def_count{}; + for (const auto& node : graph.Nodes()) + { + node.ForEachDef([&](const onnxruntime::NodeArg& /*arg*/, bool /*is_input*/) { + ++total_def_count; + }); + } + node_values_.reserve(total_def_count); + for (auto& node : graph.Nodes()) { ORT_ENFORCE(node.Index() < node_offsets_.size()); node_offsets_[node.Index()] = static_cast(node_values_.size()); @@ -451,9 +460,9 @@ static inline void VerifyShape(const MLValue* p_mlvalue, if (p_mlvalue->IsTensor()) { const Tensor* tensor = &p_mlvalue->Get(); - ORT_ENFORCE(tensor->Shape() == parameters.tensor_shape, + ORT_ENFORCE(tensor->Shape() == parameters.GetTensorShape(), "MLValue shape verification failed. Current shape:", tensor->Shape(), - " Requested shape:", parameters.tensor_shape); + " Requested shape:", parameters.GetTensorShape()); } } diff --git a/onnxruntime/core/framework/execution_frame.h b/onnxruntime/core/framework/execution_frame.h index 1d0a410e33ef5..67cfc88cf69f3 100644 --- a/onnxruntime/core/framework/execution_frame.h +++ b/onnxruntime/core/framework/execution_frame.h @@ -20,7 +20,19 @@ class MLValuePatternPlanner; struct MemoryPatternGroup; struct MLValueAllocationParameters { - TensorShape tensor_shape; + MLValueAllocationParameters() = default; + MLValueAllocationParameters(const TensorShape* shape) + : tensor_shape{ shape } + {} + + const TensorShape& GetTensorShape() const + { + static const TensorShape s_empty_tensor_shape; + return tensor_shape != nullptr ? *tensor_shape : s_empty_tensor_shape; + } + +private: + const TensorShape* tensor_shape{}; // todo: is there any parameter needed for ml types? }; diff --git a/onnxruntime/core/framework/graph_partitioner.cc b/onnxruntime/core/framework/graph_partitioner.cc index 203cd365275c7..7ca96978f9960 100644 --- a/onnxruntime/core/framework/graph_partitioner.cc +++ b/onnxruntime/core/framework/graph_partitioner.cc @@ -55,6 +55,11 @@ KernelDefBuilder& BuildFusedKernelDef(KernelDefBuilder& builder, const onnxrunti } Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { + // It is a greedy partitioning algorithm per provider preferences user provided when calling ONNX RUNTIME right now. + // 1. Execution providers' capabilities are checked one by one. + // 2. All sub-graphs that an execution provider returns will be assigned to it if it's not assigned yet. + // 3. CPU execution provider is expected to be able to run any node and is the last one in execution provider preference. + if (providers_.Empty()) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No provider specified."); } @@ -63,10 +68,17 @@ Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { std::shared_ptr fused_kernel_registry = std::make_shared(); // Partitioning based on provider preference and their capabilities. auto kernel_registries = kernel_registry_mgr_.GetAllKernelRegistries(); + + std::vector>> capabilities_of_all_providers; + GraphViewer graph_viewer(graph); + for (auto& provider : providers_) { + capabilities_of_all_providers.push_back(provider->GetCapability(graph_viewer, kernel_registries)); + } + + int i = 0; for (auto& provider : providers_) { - auto capability_results = provider->GetCapability(GraphViewer(graph), kernel_registries); int count = 0; - for (auto& capability : capability_results) { + for (auto& capability : capabilities_of_all_providers[i++]) { if (nullptr == capability || nullptr == capability->sub_graph) { continue; } @@ -78,30 +90,44 @@ Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { auto node = graph.GetNode(capability->sub_graph->nodes[0]); if (nullptr != node && node->GetExecutionProviderType().empty()) { + // The node was not fused or assigned. Assign it to this . node->SetExecutionProviderType(provider->Type()); } } else { // The can run a fused in the . - // - // Add fused node into ORT_ENFORCE(nullptr != capability->sub_graph->GetMetaDef()); - std::string node_name = provider->Type() + "_" + capability->sub_graph->GetMetaDef()->name + "_" + std::to_string(count++); - auto& fused_node = graph.FuseSubGraph(std::move(capability->sub_graph), node_name); - fused_node.SetExecutionProviderType(provider->Type()); - auto fused_kernel_func = capability->fuse_kernel_function; - if (fused_kernel_func != nullptr) { - // build the kernel definition on the fly, and register it to the fused_kernel_regisitry. - KernelDefBuilder builder; - BuildFusedKernelDef(builder, fused_node); - fused_kernel_registry->Register(builder, fused_kernel_func); + + // Check whether any node in the was already assigned. + bool sub_graph_available_for_assignment = true; + for (auto node_index : capability->sub_graph->nodes) { + auto node = graph.GetNode(node_index); + if (nullptr == node || !node->GetExecutionProviderType().empty()) { + // The node was fused or assigned, so that the whole sub-graph will not be assigned to this + // The assumption is that this can only run the sub-graph as a whole unit. + sub_graph_available_for_assignment = false; + break; + } + } + + if (sub_graph_available_for_assignment) { + // Add fused node into + std::string node_name = provider->Type() + "_" + capability->sub_graph->GetMetaDef()->name + "_" + std::to_string(count++); + auto& fused_node = graph.FuseSubGraph(std::move(capability->sub_graph), node_name); + fused_node.SetExecutionProviderType(provider->Type()); + auto fused_kernel_func = capability->fuse_kernel_function; + if (fused_kernel_func != nullptr) { + // build the kernel definition on the fly, and register it to the fused_kernel_regisitry. + KernelDefBuilder builder; + BuildFusedKernelDef(builder, fused_node); + fused_kernel_registry->Register(builder, fused_kernel_func); + } } } } - // all done with this provider, resolve the graph before we move on to the next provider. - // This is needed since we create a new GraphViewer() that we pass into the next provider's GetCapability(). - ORT_ENFORCE(graph.Resolve().IsOK()); } + ORT_ENFORCE(graph.Resolve().IsOK()); + // To see if the node with no provider can be inlined. If one such nodes can be // successfully inlined, we re-run the partitioner on the modified graph. bool inline_flag = false; @@ -126,10 +152,10 @@ Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { this->Partition(graph); } - //For some cases, like fp16 on cpu, right now we don't have any kernel support that. - //But we will insert cast op to run the model, so skip the error checking here. - //If after graph transform phase, the node still not assigned, we will report error - //during kernel creation phase. + //For some cases, like fp16 on cpu, right now we don't have any kernel support that. + //But we will insert cast op to run the model, so skip the error checking here. + //If after graph transform phase, the node still not assigned, we will report error + //during kernel creation phase. #ifdef COUNT_NON_CUDA_OPS for (auto& node : graph.Nodes()) { if (node.GetExecutionProviderType() != kCudaExecutionProvider && diff --git a/onnxruntime/core/framework/kernel_def_builder.cc b/onnxruntime/core/framework/kernel_def_builder.cc index 8555adc7563de..12d0fd936897c 100644 --- a/onnxruntime/core/framework/kernel_def_builder.cc +++ b/onnxruntime/core/framework/kernel_def_builder.cc @@ -66,20 +66,20 @@ bool KernelDef::IsConflict(const KernelDef& other) const { return false; //check memory type - auto other_input_mem_types = other.InputMemoryType(); + auto& other_input_mem_types = other.input_memory_type_args_; for (auto it : input_memory_type_args_) { - if (other_input_mem_types.count(it.first) && other_input_mem_types[it.first] == it.second) + if (other_input_mem_types.count(it.first) && other_input_mem_types.find(it.first)->second == it.second) return false; } - if (input_memory_type_args_.empty() && !other.InputMemoryType().empty()) + if (input_memory_type_args_.empty() && !other.input_memory_type_args_.empty()) return false; - auto other_output_mem_types = other.OutputMemoryType(); + auto& other_output_mem_types = other.output_memory_type_args_; for (auto it : output_memory_type_args_) { - if (other_output_mem_types.count(it.first) && other_output_mem_types[it.first] == it.second) + if (other_output_mem_types.count(it.first) && other_output_mem_types.find(it.second)->second == it.second) return false; } - return !(output_memory_type_args_.empty() && !other.OutputMemoryType().empty()); + return !(output_memory_type_args_.empty() && !other.output_memory_type_args_.empty()); } KernelDefBuilder& KernelDefBuilder::SetName(const std::string& op_name) { diff --git a/onnxruntime/core/framework/onnx_object.cc b/onnxruntime/core/framework/onnx_object.cc index d7232e477c8d7..199b68dc5827b 100644 --- a/onnxruntime/core/framework/onnx_object.cc +++ b/onnxruntime/core/framework/onnx_object.cc @@ -5,17 +5,17 @@ #include ORT_API(uint32_t, OrtAddRefToObject, void* ptr) { - return (*static_cast(ptr))->AddRef(ptr); + return (*static_cast(ptr))->AddRef(ptr); } ORT_API(uint32_t, OrtReleaseObject, void* ptr) { if (ptr == nullptr) return 0; - return (*static_cast(ptr))->Release(ptr); + return (*static_cast(ptr))->Release(ptr); } namespace { struct ObjectImpl { - const ONNXObject* const cls; + const OrtObject* const cls; std::atomic_int ref_count; }; } // namespace diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.cc b/onnxruntime/core/framework/onnxruntime_typeinfo.cc index 0693daa2f4ff1..655bc840fceb0 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.cc +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.cc @@ -13,7 +13,7 @@ using onnxruntime::MLFloat16; using onnxruntime::Tensor; using onnxruntime::TensorShape; -OrtTypeInfo::OrtTypeInfo(OrtType type1, void* data1) noexcept : type(type1), data(data1) { +OrtTypeInfo::OrtTypeInfo(ONNXType type1, void* data1) noexcept : type(type1), data(data1) { } OrtTypeInfo::~OrtTypeInfo() { @@ -22,37 +22,37 @@ OrtTypeInfo::~OrtTypeInfo() { } ORT_API(const struct OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ struct OrtTypeInfo* input) { - return input->type == ORT_TYPE_TENSOR ? reinterpret_cast(input->data) : nullptr; + return input->type == ONNX_TYPE_TENSOR ? reinterpret_cast(input->data) : nullptr; } -ONNXStatus* GetTensorShapeAndType(const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out); +OrtStatus* GetTensorShapeAndType(const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out); -ONNXStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out) { +OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out) { if (input == nullptr) { - *out = new OrtTypeInfo(ORT_TYPE_UNKNOWN, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_UNKNOWN, nullptr); return nullptr; } if (input == DataTypeImpl::GetType()) { OrtTensorTypeAndShapeInfo* info = nullptr; if (tensor_data_type != nullptr) { - ONNXStatus* st = GetTensorShapeAndType(shape, tensor_data_type, &info); + OrtStatus* st = GetTensorShapeAndType(shape, tensor_data_type, &info); if (st != nullptr) return st; } - *out = new OrtTypeInfo(ORT_TYPE_TENSOR, info); + *out = new OrtTypeInfo(ONNX_TYPE_TENSOR, info); return nullptr; } if (input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType()) { - *out = new OrtTypeInfo(ORT_TYPE_MAP, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_MAP, nullptr); return nullptr; } if (input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType()) { - *out = new OrtTypeInfo(ORT_TYPE_SEQUENCE, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_SEQUENCE, nullptr); return nullptr; } - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, "not implemented"); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); } -const DataTypeImpl* ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { +const DataTypeImpl* ElementTypeFromProto(int type) { switch (type) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: return DataTypeImpl::GetType(); @@ -85,11 +85,11 @@ const DataTypeImpl* ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType ty } } -ONNXStatus* OrtTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, OrtTypeInfo** out) { +OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, OrtTypeInfo** out) { if (input->has_tensor_type()) { const ::onnx::TypeProto_Tensor& onnx_tensor_info = input->tensor_type(); const DataTypeImpl* type = ElementTypeFromProto(onnx_tensor_info.elem_type()); - ONNXStatus* st; + OrtStatus* st; OrtTensorTypeAndShapeInfo* info = nullptr; if (onnx_tensor_info.has_shape()) { const ::onnx::TensorShapeProto& s = onnx_tensor_info.shape(); @@ -104,24 +104,24 @@ ONNXStatus* OrtTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, OrtTypeI } if (st != nullptr) return st; - *out = new OrtTypeInfo(ORT_TYPE_TENSOR, info); + *out = new OrtTypeInfo(ONNX_TYPE_TENSOR, info); return nullptr; } if (input->has_sequence_type()) { - *out = new OrtTypeInfo(ORT_TYPE_SEQUENCE, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_SEQUENCE, nullptr); return nullptr; } if (input->has_map_type()) { - *out = new OrtTypeInfo(ORT_TYPE_MAP, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_MAP, nullptr); return nullptr; } if (input->has_opaque_type()) { - *out = new OrtTypeInfo(ORT_TYPE_OPAQUE, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_OPAQUE, nullptr); return nullptr; } if (input->has_sparse_tensor_type()) { - *out = new OrtTypeInfo(ORT_TYPE_SPARSETENSOR, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_SPARSETENSOR, nullptr); return nullptr; } - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, "not implemented"); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); } diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.h b/onnxruntime/core/framework/onnxruntime_typeinfo.h index 7c2f5331bc76c..9cbef90d24bde 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.h +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.h @@ -22,17 +22,17 @@ struct OrtTypeInfo : public onnxruntime::ObjectBase { public: friend class onnxruntime::ObjectBase; - OrtType type = ORT_TYPE_UNKNOWN; + ONNXType type = ONNX_TYPE_UNKNOWN; //owned by this void* data = nullptr; OrtTypeInfo(const OrtTypeInfo& other) = delete; OrtTypeInfo& operator=(const OrtTypeInfo& other) = delete; - static ONNXStatus* FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const onnxruntime::TensorShape* shape, + static OrtStatus* FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out); - static ONNXStatus* FromDataTypeImpl(const onnx::TypeProto*, OrtTypeInfo** out); + static OrtStatus* FromDataTypeImpl(const onnx::TypeProto*, OrtTypeInfo** out); private: - OrtTypeInfo(OrtType type, void* data) noexcept; + OrtTypeInfo(ONNXType type, void* data) noexcept; ~OrtTypeInfo(); }; diff --git a/onnxruntime/core/framework/op_kernel.cc b/onnxruntime/core/framework/op_kernel.cc index ad1a6d11498d1..65792961197b7 100644 --- a/onnxruntime/core/framework/op_kernel.cc +++ b/onnxruntime/core/framework/op_kernel.cc @@ -29,8 +29,8 @@ Tensor* OpKernelContext::Output(int index, const TensorShape& shape) { // In this case, it's assumed that the tensor hasn't been allocated yet, // so that it's calling ExecutionFrame to create a tensor in the given position with given shape. - MLValueAllocationParameters parameters; - parameters.tensor_shape = shape; + MLValueAllocationParameters parameters{ &shape }; + //: Though we don't need to give 'ret' an initial value, GCC would generate a warning if we don't do that //"error: 'ret' may be used uninitialized in this function" //This warning only exists in Release build. diff --git a/onnxruntime/core/framework/session_state_initializer.cc b/onnxruntime/core/framework/session_state_initializer.cc index 6d8df8500ed80..4c2126c48c244 100644 --- a/onnxruntime/core/framework/session_state_initializer.cc +++ b/onnxruntime/core/framework/session_state_initializer.cc @@ -26,12 +26,6 @@ namespace onnxruntime { -static common::Status TransformGraph(onnxruntime::Graph& graph, - const onnxruntime::GraphTransformerManager& graph_transformer_mgr, - const ExecutionProviders& exec_providers, - KernelRegistryManager& kernel_registry_manager, - const InsertCastTransformer& insert_cast_transformer); - static common::Status SaveMLValueNameIndexMapping(const onnxruntime::Graph& graph, MLValueNameIdxMap& mlvalue_name_idx_map, const logging::Logger& logger); @@ -68,15 +62,9 @@ SessionStateInitializer::SessionStateInitializer(onnxruntime::Graph& graph, logger_{logger} { } -common::Status SessionStateInitializer::CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, - const InsertCastTransformer& insert_cast_transformer, - const std::vector& outer_scope_node_args, +common::Status SessionStateInitializer::CreatePlan(const std::vector& outer_scope_node_args, bool enable_sequential_execution) { - ORT_RETURN_IF_ERROR(TransformGraph(graph_, graph_transformation_manager, - execution_providers_, kernel_registry_manager_, - insert_cast_transformer)); - - // After transformation/partitioning, the graph now is fixed and graph viewer is created and set for execution. + // the graph now is fixed and graph viewer is created and set for execution. session_state_.SetGraphViewer(std::make_unique(graph_)); auto& mlvalue_name_idx_map = session_state_.GetMLValueNameIdxMap(); @@ -142,46 +130,6 @@ common::Status SessionStateInitializer::InitializeAndSave(bool enable_memory_pat return Status::OK(); } -common::Status TransformGraph(onnxruntime::Graph& graph, - const onnxruntime::GraphTransformerManager& graph_transformer_mgr, - const ExecutionProviders& providers, - KernelRegistryManager& kernel_registry_manager, - const InsertCastTransformer& insert_cast_transformer) { - // The transformer order: - // 1. built-in graph rewriter - // 2. each execution provider's transformer - // 3. do node placement according to kernel definition - // 4. insert copy nodes - // 5. insert cast nodes. - - // first apply the default/system/basic graph to graph optimizations. - ORT_RETURN_IF_ERROR(graph_transformer_mgr.ApplyAll(graph)); - - auto kernels{kernel_registry_manager.GetAllKernelRegistries()}; - - // Do partitioning based on execution providers' capability. - GraphPartitioner partitioner(kernel_registry_manager, providers); - ORT_RETURN_IF_ERROR(partitioner.Partition(graph)); - - // Insert copy nodes. - for (auto& provider : providers) { - if (provider->Type() != onnxruntime::kCpuExecutionProvider && - provider->Type() != onnxruntime::kMklDnnExecutionProvider && - provider->Type() != onnxruntime::kNupharExecutionProvider) { - TransformerMemcpyImpl copy_impl(graph, provider->Type()); - copy_impl.ModifyGraph(kernel_registry_manager); - } - } - - // Insert cast node/s. - bool modified = false; - ORT_RETURN_IF_ERROR(insert_cast_transformer.Apply(graph, modified)); - - ORT_RETURN_IF_ERROR(graph.Resolve()); - - return common::Status::OK(); -} - // Build the MLValue name->idx mapping common::Status SaveMLValueNameIndexMapping(const onnxruntime::Graph& graph, MLValueNameIdxMap& mlvalue_name_idx_map, diff --git a/onnxruntime/core/framework/session_state_initializer.h b/onnxruntime/core/framework/session_state_initializer.h index 95fc97d65b3ab..fa8e4b5f98847 100644 --- a/onnxruntime/core/framework/session_state_initializer.h +++ b/onnxruntime/core/framework/session_state_initializer.h @@ -29,9 +29,7 @@ class SessionStateInitializer { const logging::Logger& logger); // First perform any transformations and create the execution plan - common::Status CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, - const InsertCastTransformer& insert_cast_transformer, - const std::vector& outer_scope_node_args, + common::Status CreatePlan(const std::vector& outer_scope_node_args, bool enable_sequential_execution); // initialize tensors, and save. save kernels and input/output node mappings diff --git a/onnxruntime/core/framework/tensor_type_and_shape.cc b/onnxruntime/core/framework/tensor_type_and_shape.cc index 76f0c88957daa..eb22ef015c12e 100644 --- a/onnxruntime/core/framework/tensor_type_and_shape.cc +++ b/onnxruntime/core/framework/tensor_type_and_shape.cc @@ -18,7 +18,7 @@ struct OrtTensorTypeAndShapeInfo : public onnxruntime::ObjectBase; - OrtTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; onnxruntime::TensorShape shape; static OrtTensorTypeAndShapeInfo* Create() { @@ -39,14 +39,14 @@ struct OrtTensorTypeAndShapeInfo : public onnxruntime::ObjectBasetype = type; return nullptr; @@ -60,7 +60,7 @@ ORT_API_STATUS_IMPL(OrtSetDims, _In_ OrtTensorTypeAndShapeInfo* this_ptr, _In_ c API_IMPL_END } -ORT_API(enum OrtTensorElementDataType, OrtGetTensorElementType, _In_ const struct OrtTensorTypeAndShapeInfo* info) { +ORT_API(enum ONNXTensorElementDataType, OrtGetTensorElementType, _In_ const struct OrtTensorTypeAndShapeInfo* info) { return info->type; } @@ -76,12 +76,12 @@ ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const OrtTensorTypeAndShape return this_ptr->shape.Size(); } -struct ONNXValue; +struct OrtValue; namespace { -inline OrtTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( +inline ONNXTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( const onnxruntime::DataTypeImpl* cpp_type) { - OrtTensorElementDataType type; + ONNXTensorElementDataType type; if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { @@ -115,10 +115,10 @@ inline OrtTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( } } // namespace -ONNXStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) { - OrtTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type); +OrtStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) { + ONNXTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type); if (ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED == type) { - return CreateONNXStatus(ORT_FAIL, "Not implemented"); + return OrtCreateStatus(ORT_FAIL, "Not implemented"); } OrtTensorTypeAndShapeInfo* ret = OrtCreateTensorTypeAndShapeInfo(); auto status = OrtSetTensorElementType(ret, type); @@ -137,7 +137,7 @@ ONNXStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const o return nullptr; } -ORT_API_STATUS_IMPL(OrtGetTensorShapeAndType, _In_ const ONNXValue* value, +ORT_API_STATUS_IMPL(OrtGetTensorShapeAndType, _In_ const OrtValue* value, _Out_ OrtTensorTypeAndShapeInfo** out) { API_IMPL_BEGIN auto v = reinterpret_cast(value); @@ -146,30 +146,30 @@ ORT_API_STATUS_IMPL(OrtGetTensorShapeAndType, _In_ const ONNXValue* value, API_IMPL_END } -ORT_API(enum OrtType, OrtGetValueType, _In_ const ONNXValue* value) { +ORT_API(enum ONNXType, OrtGetValueType, _In_ const OrtValue* value) { try { auto v = reinterpret_cast(value); onnxruntime::MLDataType type = v->Type(); OrtTypeInfo* out; - ONNXStatus* ptr = OrtTypeInfo::FromDataTypeImpl(type, nullptr, nullptr, &out); + OrtStatus* ptr = OrtTypeInfo::FromDataTypeImpl(type, nullptr, nullptr, &out); if (ptr != nullptr) { - ReleaseONNXStatus(ptr); - return ORT_TYPE_UNKNOWN; + OrtReleaseStatus(ptr); + return ONNX_TYPE_UNKNOWN; } - OrtType ret = out->type; + ONNXType ret = out->type; OrtReleaseObject(out); return ret; } catch (std::exception&) { - return ORT_TYPE_UNKNOWN; + return ONNX_TYPE_UNKNOWN; } } /** - * Get the type information of an ONNXValue + * Get the type information of an OrtValue * \param value * \return The returned value should be freed by OrtReleaseObject after use */ -ORT_API_STATUS_IMPL(OrtGetTypeInfo, _In_ const ONNXValue* value, struct OrtTypeInfo** out) { +ORT_API_STATUS_IMPL(OrtGetTypeInfo, _In_ const OrtValue* value, struct OrtTypeInfo** out) { auto v = reinterpret_cast(value); onnxruntime::MLDataType type = v->Type(); if (type == nullptr) { diff --git a/onnxruntime/core/framework/transformer_memcpy.cc b/onnxruntime/core/framework/transformer_memcpy.cc index a4faef010d5c0..b04f278599324 100644 --- a/onnxruntime/core/framework/transformer_memcpy.cc +++ b/onnxruntime/core/framework/transformer_memcpy.cc @@ -68,25 +68,24 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg // note KernelCreateInfo might be nullptr for custom kernel const KernelCreateInfo* kci = nullptr; kernel_registries.SearchKernelRegistry(node, &kci); - const auto* input_mem_types = kci ? &kci->kernel_def->InputMemoryType() : nullptr; - const auto* output_mem_types = kci ? &kci->kernel_def->InputMemoryType() : nullptr; + ORT_ENFORCE(onnxruntime::Node::ForEachWithIndex( - node.InputDefs(), - [this, &input_mem_types](const onnxruntime::NodeArg& arg, size_t index) { - if (input_mem_types && MemTypeOnCpuExplicitly(*input_mem_types, index)) - non_provider_input_defs_.insert(&arg); - else - provider_input_defs_.insert(&arg); - return Status::OK(); - }) - .IsOK()); + node.InputDefs(), + [this, &kci](const onnxruntime::NodeArg& arg, size_t index) { + if (kci && MemTypeOnCpuExplicitly(kci->kernel_def->InputMemoryType(index))) + non_provider_input_defs_.insert(&arg); + else + provider_input_defs_.insert(&arg); + return Status::OK(); + }) + .IsOK()); auto& output_defs = node.MutableOutputDefs(); for (size_t i = 0; i < output_defs.size(); ++i) { auto arg = output_defs[i]; if (!arg->Exists()) continue; - if (output_mem_types && MemTypeOnCpuExplicitly(*output_mem_types, i)) + if (kci && MemTypeOnCpuExplicitly(kci->kernel_def->OutputMemoryType(i))) non_provider_output_defs_.insert(arg); else provider_output_defs_.insert(arg); diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 7316b85235b8a..9a2e58057c278 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -8,6 +8,10 @@ #include "core/graph/op.h" #include "onnx/defs/shape_inference.h" +#ifdef MICROSOFT_INTERNAL +#include "core/graph/contrib_ops/internal_schema_defs.h" +#endif + namespace ONNX_NAMESPACE { void convPoolTypeAndShapeInference(ONNX_NAMESPACE::InferenceContext& ctx, bool use_dilation, bool require_kernel_shape); } @@ -588,6 +592,75 @@ The bounding box coordinates corresponding to the selected indices can then be o output_elem_type->set_elem_type(ONNX_NAMESPACE::TensorProto::STRING); }) .SetDoc(R"DOC([optional] Step1: Remove elements in X if they match any of the stop words so that the output tensor will not contain any stop words. This operator only accepts [C]- and [1, C]-tensors. If all elements in X are dropped, the output will be the default value of string tensor with shape [1] if input shape is [C] and shape [1, 1] if input shape is [1, C].)DOC"); + + ONNX_CONTRIB_OPERATOR_SCHEMA(GatherND) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input(0, "data", "Tensor of rank r >= 1.", "T") + .Input(1, "indices", "Tensor of rank q >= 1.", "Tind") + .Output(0, "output", "Tensor of rank q-1+r-indices[-1].", "T") + .TypeConstraint( + "T", + OpSchema::all_tensor_types(), + "Constrain input and output types to any tensor type.") + .TypeConstraint( + "Tind", + {"tensor(int32)", "tensor(int64)"}, + "Constrain indice type to int32 or int64") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + if (!hasNInputShapes(ctx, 2)) { + fail_shape_inference("GatherND requires two tensor inputs."); + } + auto& data_shape = ctx.getInputType(0)->tensor_type().shape(); + auto& indices_shape = ctx.getInputType(1)->tensor_type().shape(); + auto data_rank = data_shape.dim_size(); + auto indices_rank = indices_shape.dim_size(); + if (data_rank < 1 || indices_rank < 1) { + fail_shape_inference("both data and indices tensor need to have rank larger than zero."); + } + auto last_indice_dimension = indices_shape.dim(indices_rank - 1).dim_value(); + if (last_indice_dimension > data_rank) { + fail_shape_inference("last dimension of indices must not be larger and rank of data tensor"); + } + for (int i = 0; i < indices_rank - 1; ++i) { + *ctx.getOutputType(0) + ->mutable_tensor_type() + ->mutable_shape() + ->add_dim() = indices_shape.dim(i); + } + for (int i = static_cast(last_indice_dimension); i < data_rank; ++i) { + *ctx.getOutputType(0) + ->mutable_tensor_type() + ->mutable_shape() + ->add_dim() = data_shape.dim(i); + } + }) + .SetDoc(R"DOC( +Given `data` tensor of rank r >= 1, and `indices` tensor of rank q >= 1, gather +slices of `data` into an output tensor of rank q - 1 + r - indices[-1]. +Example 1: + data = [[0,1],[2,3]] + indices = [[0,0],[1,1]] + output = [0,3] +Example 2: + data = [[0,1],[2,3]] + indices = [[1],[0]] + output = [[2,3],[0,1]] +Example 3: + data = [[[0,1],[2,3]],[[4,5],[6,7]]] + indices = [[0,1],[1,0]] + output = [[2,3],[4,5]] +Example 4: + data = [[[0,1],[2,3]],[[4,5],[6,7]]] + indices = [[[0,1]],[[1,0]]] + output = [[[2,3]],[[4,5]]] +)DOC"); + +#ifdef MICROSOFT_INTERNAL + // register internal ops + RegisterInternalSchemas(); +#endif } } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc b/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc index fc31ee4f94371..e5524281b223e 100644 --- a/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc @@ -81,7 +81,7 @@ static int64_t CalcRangeDim(const TensorProto* startShapeInitializer, static int64_t CalcResultDim(const TensorProto* startShapeInitializer, const TensorProto* limitShapeInitializer, const TensorProto* deltaShapeInitializer, - TensorProto_DataType dtype) { + int dtype) { int64_t dim = -1LL; if (dtype == TensorProto::FLOAT) { dim = CalcRangeDim(startShapeInitializer, limitShapeInitializer, deltaShapeInitializer); @@ -146,7 +146,7 @@ OpSchema& RegisterRangeOpSchema(OpSchema&& op_schema){ const TensorProto* limitShapeInitializer = ctx.getInputData(1); const TensorProto* deltaShapeInitializer = (ctx.getNumInputs() > 2) ? ctx.getInputData(2) : nullptr; const auto& startTensorType = ctx.getInputType(0)->tensor_type(); - TensorProto_DataType dtype = startTensorType.elem_type(); + int dtype = startTensorType.elem_type(); int64_t n = CalcResultDim(startShapeInitializer, limitShapeInitializer, deltaShapeInitializer, dtype); dim.set_dim_value(n); diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index b593e0b5bb656..38b2df4c78d8d 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -152,8 +152,8 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu if (input_tensor_elem_type != current_tensor_elem_type) return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Tensor element type mismatch. ", - TensorProto_DataType_Name(input_tensor_elem_type), " != ", - TensorProto_DataType_Name(current_tensor_elem_type)); + TensorProto_DataType_Name(static_cast(input_tensor_elem_type)), " != ", + TensorProto_DataType_Name(static_cast(current_tensor_elem_type))); if (input_tensor_type.has_shape()) { auto& current_tensor_type = *current_type.mutable_tensor_type(); @@ -172,8 +172,8 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu const auto current_tensor_elem_type = current_type.sparse_tensor_type().elem_type(); if (input_tensor_elem_type != current_tensor_elem_type) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "SparseTensor element type mismatch. ", - TensorProto_DataType_Name(input_tensor_elem_type), " != ", - TensorProto_DataType_Name(current_tensor_elem_type)); + TensorProto_DataType_Name(static_cast(input_tensor_elem_type)), " != ", + TensorProto_DataType_Name(static_cast(current_tensor_elem_type))); } if (input_tensor_type.has_shape()) { auto& current_tensor_type = *current_type.mutable_sparse_tensor_type(); @@ -830,7 +830,6 @@ void Graph::AddEdge(NodeIndex src_node_index, NodeIndex dst_node_index, int src_ // The output type of source node arg does not match the input type of destination node arg. ORT_THROW("Argument type mismatch when adding edge."); } else { - src_arg->UpdateTypeAndShape(*dst_arg); *dst_arg_pointer = src_arg; } } @@ -892,12 +891,12 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c subgraph->BuildConnections(node_args_consumed); for (auto& node_arg_name : node_args_consumed) { - bool node_arg_in_parent_graph = false; auto node_arg = GetNodeArg(node_arg_name); if (node_arg == nullptr) { // it's a node arg from outside this graph's scope, so add that to the list we return - // so that we can add the dependency at the next level up + // so that we can add the dependency at the next level up. this happens if you have multiple + // levels of subgraphs between the graph with the original NodeArg and the subgraph with implicit usage. outer_scope_node_args_consumed.push_back(node_arg_name); if (!parent_graph_) { @@ -917,8 +916,6 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c "Failed to find NodeArg in all parent graphs. Name=", node_arg_name, " Graph may not conform to the ONNX spec and contain initializers that are not graph inputs."); } - - node_arg_in_parent_graph = true; } // add it to the Node's list of implicit inputs @@ -932,18 +929,8 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c input_slot_index += static_cast(iter - implicit_inputs.cbegin()); } - if (node_arg_in_parent_graph || - resolve_context_.inputs_and_initializers.find(node_arg_name) != - resolve_context_.inputs_and_initializers.cend()) { - // no connection required if it's an input or initializer. - // if the node arg is from a parent graph we link the nodes in the parent graph by passing the - // node_arg_name back up in outer_scope_node_args_consumed - - } else { - // if it's an output nodearg in this graph we need to create a link to the node the output is coming from - auto entry = resolve_context_.output_args.find(node_arg_name); - ORT_ENFORCE(entry != resolve_context_.output_args.end()); - + auto entry = resolve_context_.output_args.find(node_arg_name); + if (entry != resolve_context_.output_args.end()) { // Create relationship between this node (node), and the node providing the output (output_node). Node& output_node = *entry->second.first; AddEdge(output_node.Index(), node->Index(), entry->second.second, input_slot_index); @@ -1650,7 +1637,7 @@ Status Graph::VerifyNodeAndOpMatch() { auto maxInclusiveVersion = DomainToVersionMap().find(domain)->second; node.op_ = schema_registry_->GetSchema(node.OpType(), maxInclusiveVersion, node.Domain()); - if (node.op_->Deprecated()) { + if (node.op_ && node.op_->Deprecated()) { node.op_ = nullptr; } diff --git a/onnxruntime/core/graph/graph_utils.cc b/onnxruntime/core/graph/graph_utils.cc index 7daeba838839d..3dcbc71358136 100644 --- a/onnxruntime/core/graph/graph_utils.cc +++ b/onnxruntime/core/graph/graph_utils.cc @@ -4,18 +4,70 @@ namespace onnxruntime { namespace utils { - // fusion is only done for ONNX domain ops - bool IsSupportedOptypeVersionAndDomain(const Node& node, - const std::string& op_type, - ONNX_NAMESPACE::OperatorSetVersion version, - const std::string& domain) { - if (node.OpType() != op_type || - node.Op()->Deprecated() || node.Op()->SinceVersion() != version || - (!node.Domain().empty() && node.Domain() != domain)) { - return false; +// fusion is only done for ONNX domain ops +bool IsSupportedOptypeVersionAndDomain(const Node& node, + const std::string& op_type, + ONNX_NAMESPACE::OperatorSetVersion version, + const std::string& domain) { + if (node.OpType() != op_type || + node.Op()->Deprecated() || node.Op()->SinceVersion() != version || + (!node.Domain().empty() && node.Domain() != domain)) { + return false; + } + return true; +} + +Status ForAllMutableSubgraphs(Graph& graph, std::function func) { + Status status = Status::OK(); + + for (auto& node : graph.Nodes()) { + for (auto& attribute : node.GetAttributes()) { + auto& name = attribute.first; + auto& proto = attribute.second; + + // check if it has a subgraph + if (proto.has_g()) { + Graph* subgraph = node.GetMutableGraphAttribute(name); + ORT_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); + + status = func(*subgraph); + ORT_RETURN_IF_ERROR(status); + + // recurse + status = ForAllMutableSubgraphs(*subgraph, func); + ORT_RETURN_IF_ERROR(status); + } } - return true; } + + return status; +} + +Status ForAllSubgraphs(const Graph& graph, std::function func) { + Status status = Status::OK(); + + for (auto& node : graph.Nodes()) { + for (auto& attribute : node.GetAttributes()) { + auto& name = attribute.first; + auto& proto = attribute.second; + + // check if it has a subgraph + if (proto.has_g()) { + const Graph* subgraph = node.GetGraphAttribute(name); + ORT_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); + + status = func(*subgraph); + ORT_RETURN_IF_ERROR(status); + + // recurse + status = ForAllSubgraphs(*subgraph, func); + ORT_RETURN_IF_ERROR(status); + } + } + } + + return status; } -} // namespace onnxruntime \ No newline at end of file +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph_utils.h b/onnxruntime/core/graph/graph_utils.h index 7ff5e93d3e09a..993a535bab0ff 100644 --- a/onnxruntime/core/graph/graph_utils.h +++ b/onnxruntime/core/graph/graph_utils.h @@ -9,10 +9,13 @@ namespace onnxruntime { namespace utils { - bool IsSupportedOptypeVersionAndDomain(const Node& node, - const std::string& op_type, - ONNX_NAMESPACE::OperatorSetVersion version, - const std::string& domain = kOnnxDomainAlias); -} +bool IsSupportedOptypeVersionAndDomain(const Node& node, + const std::string& op_type, + ONNX_NAMESPACE::OperatorSetVersion version, + const std::string& domain = kOnnxDomainAlias); -} \ No newline at end of file +Status ForAllMutableSubgraphs(Graph& main_graph, std::function func); +Status ForAllSubgraphs(Graph& main_graph, std::function func); + +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/initializer.h b/onnxruntime/core/graph/initializer.h index 08b7951acda61..d6c15b2cd49ea 100644 --- a/onnxruntime/core/graph/initializer.h +++ b/onnxruntime/core/graph/initializer.h @@ -128,11 +128,11 @@ class Initializer final { } } - ONNX_NAMESPACE::TensorProto_DataType data_type() const { + int data_type() const { return data_type_; } - ONNX_NAMESPACE::TensorProto_DataType& data_type() { + int& data_type() { return data_type_; } @@ -372,7 +372,7 @@ class Initializer final { } private: - ONNX_NAMESPACE::TensorProto_DataType data_type_; + int data_type_; std::string name_; std::vector dims_; int64_t size_; diff --git a/onnxruntime/core/protobuf/onnx-ml.proto b/onnxruntime/core/protobuf/onnx-ml.proto index 79edbb6ef49ce..10463ddf63409 100644 --- a/onnxruntime/core/protobuf/onnx-ml.proto +++ b/onnxruntime/core/protobuf/onnx-ml.proto @@ -330,7 +330,7 @@ message TensorProto { repeated int64 dims = 1; // The data type of the tensor. - optional DataType data_type = 2; + optional int32 data_type = 2; // For very large tensors, we may want to store them in chunks, in which // case the following fields will specify the segment that is stored in @@ -438,7 +438,7 @@ message TypeProto { message Tensor { // This field MUST NOT have the value of UNDEFINED // This field MUST be present for this version of the IR. - optional TensorProto.DataType elem_type = 1; + optional int32 elem_type = 1; optional TensorShapeProto shape = 2; } @@ -454,7 +454,7 @@ message TypeProto { message Map { // This field MUST be present for this version of the IR. // This field MUST refer to an integral type ([U]INT{8|16|32|64}) or STRING - optional TensorProto.DataType key_type = 1; + optional int32 key_type = 1; // This field MUST be present for this version of the IR. optional TypeProto value_type = 2; }; @@ -469,10 +469,10 @@ message TypeProto { // repeated TypeProto parameters = 3; } - message SparseTensor { - // This field MUST NOT have the value of UNDEFINED - // This field MUST be present for this version of the IR. - optional TensorProto.DataType elem_type = 1; + message SparseTensor { + // This field MUST NOT have the value of UNDEFINED + // This field MUST be present for this version of the IR. + optional int32 elem_type = 1; optional TensorShapeProto shape = 2; } diff --git a/onnxruntime/core/providers/cpu/activation/activations.h b/onnxruntime/core/providers/cpu/activation/activations.h index 42c434a2e276a..1283f570a1121 100644 --- a/onnxruntime/core/providers/cpu/activation/activations.h +++ b/onnxruntime/core/providers/cpu/activation/activations.h @@ -90,7 +90,7 @@ class ParametricSoftplus final : public OpKernel { }; template -class Relu final : public OpKernel { +class Relu : public OpKernel { public: Relu(const OpKernelInfo& info) : OpKernel(info) {} diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 1e6318949a73e..f992316cd3108 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -196,6 +196,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Eye class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, float, IsNaN); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, MLFloat16, IsNaN); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Erf); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, MaxUnpool); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Sinh); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Cosh); void RegisterOnnxOperatorKernels(std::function fn) { fn(BuildKernel()); @@ -384,6 +387,9 @@ void RegisterOnnxOperatorKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); } // Forward declarations of ml op kernels @@ -485,7 +491,7 @@ static void RegisterCPUKernels(std::function create_fn std::shared_ptr CPUExecutionProvider::GetKernelRegistry() const { static std::shared_ptr - kernel_registry = std::make_shared(RegisterCPUKernels); + kernel_registry = std::make_shared(RegisterCPUKernels); return kernel_registry; } @@ -493,7 +499,7 @@ std::vector> CPUExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, const std::vector& kernel_registries) const { std::vector> - result = IExecutionProvider::GetCapability(graph, kernel_registries); + result = IExecutionProvider::GetCapability(graph, kernel_registries); for (auto& rule : fuse_rules_) { rule(graph, result); diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc index 98813317834a1..dc8f9532bdd9c 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc @@ -15,7 +15,7 @@ struct CpuProviderFactory { CpuProviderFactory(); }; -ONNXStatus* ORT_API_CALL CreateCpu(void* this_, OrtProvider** out) { +OrtStatus* ORT_API_CALL CreateCpu(void* this_, OrtProvider** out) { CPUExecutionProviderInfo info; CpuProviderFactory* this_ptr = (CpuProviderFactory*)this_; info.create_arena = this_ptr->create_arena; diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc index f2063f5736a24..1f8324fbefe6e 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc @@ -400,11 +400,10 @@ Status Pow::Compute(OpKernelContext* context) const { std::function, ConstEigenVectorMap, float)> input1scalar = [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = Eigen::pow(input0.array(), input1); }; if (Y.Shape().Size() == 1) { - float value = * Y.Data(); + float value = *Y.Data(); if (value == 2.0) { input1scalar = [](EigenVectorMap output, ConstEigenVectorMap input0, float) { output = Eigen::square(input0.array()); }; - } - else if (value == 3.0) { + } else if (value == 3.0) { input1scalar = [](EigenVectorMap output, ConstEigenVectorMap input0, float) { output = Eigen::cube(input0.array()); }; } } @@ -789,6 +788,46 @@ ONNX_CPU_OPERATOR_KERNEL( KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Atan); +template +class Sinh final : public OpKernel { + public: + explicit Sinh(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).sinh(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Sinh, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sinh); + +template +class Cosh final : public OpKernel { + public: + explicit Cosh(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).cosh(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Cosh, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Cosh); + template <> Status PRelu::Compute(OpKernelContext* context) const { return BroadcastTwo( @@ -887,7 +926,6 @@ Status Erf::Compute(OpKernelContext* context) const { ORT_ENFORCE(X_ptr != nullptr); auto& X = *X_ptr; auto& Y = *context->Output(0, X.Shape()); - EigenMap(Y) = EigenMap(X).array().erf(); return Status::OK(); diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.h b/onnxruntime/core/providers/cpu/math/element_wise_ops.h index f36974dec9ec2..ce384776d8d54 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.h +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.h @@ -139,7 +139,7 @@ class Log final : public OpKernel { }; template -class Sum_6 final : public OpKernel { +class Sum_6 : public OpKernel { public: Sum_6(const OpKernelInfo& info) : OpKernel(info) { } diff --git a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc index eb79c5b5a62f7..212ca2eef9abb 100644 --- a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc +++ b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc @@ -53,36 +53,47 @@ template common::Status ArrayFeatureExtractorOp::Compute(OpKernelContext* context) const { const Tensor& X = *context->Input(0); const TensorShape& x_shape = X.Shape(); - const vector& x_dims = x_shape.GetDims(); + const size_t x_num_dims = x_shape.NumDimensions(); const T* x_data = X.template Data(); - if (x_dims.empty()) { + if (x_num_dims == 0) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid argument: X input has empty dimensions."); } - int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; - int64_t N = x_dims.size() == 1 ? 1 : x_dims[0]; + const int64_t stride = x_shape[x_num_dims - 1]; const Tensor& Y = *context->Input(1); const TensorShape& y_shape = Y.Shape(); const int64_t* y_data = Y.template Data(); - int64_t num_indices = y_shape.Size(); + const int64_t num_indices = y_shape.Size(); // validate Y if (num_indices == 0) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid Y argument: num_indices = 0"); } - if (num_indices - 1 >= stride) { - std::ostringstream err_msg; - err_msg << "Invalid Y argument: num_indices - 1 (" << num_indices - 1 << ") >= stride (" << stride << ")"; - return Status(ONNXRUNTIME, INVALID_ARGUMENT, err_msg.str()); + for (int64_t i = 0; i < num_indices; ++i) { + if (y_data[i] >= stride) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid Y argument: index is out of range: Y[", i, "] (", y_data[i], ") >=", stride); + } } - Tensor* Z = context->Output(0, TensorShape({N, num_indices})); + const TensorShape z_shape = [num_indices, x_num_dims, &x_shape]() { + if (x_num_dims == 1) { + // special case: for 1D input, return {1, num_indices} for backwards compatibility + return TensorShape{1, num_indices}; + } + TensorShape shape{x_shape}; + shape[x_num_dims - 1] = num_indices; + return shape; + }(); + Tensor* Z = context->Output(0, z_shape); T* z_data = Z->template MutableData(); - for (int64_t n = 0; n < N; ++n) { + const int64_t x_size_until_last_dim = x_shape.SizeToDimension(x_num_dims - 1); + for (int64_t i = 0; i < x_size_until_last_dim; ++i) { for (int64_t j = 0; j < num_indices; ++j) { *z_data++ = x_data[y_data[j]]; } diff --git a/onnxruntime/core/providers/cpu/nn/Unpool.cc b/onnxruntime/core/providers/cpu/nn/Unpool.cc new file mode 100644 index 0000000000000..391326dbb2d61 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/Unpool.cc @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// disable warning because std::copy is used by Sliceiterator +// std::copy_n is not an option for raw pointer destinations as used by gsl::copy. +#ifdef _MSC_VER +#pragma warning(disable : 4996) +#endif +#include "core/providers/cpu/nn/unpool.h" +#include "core/providers/cpu/tensor/utils.h" +#include + +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + MaxUnpool, + 9, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("I", DataTypeImpl::GetTensorType()) + .TypeConstraint("Y", DataTypeImpl::GetTensorType()), + MaxUnpool); + +Status MaxUnpool::Compute(OpKernelContext* context) const { + // Get pooled values tensor + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + const float* X_data = X->template Data(); + + ORT_RETURN_IF_NOT(X_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); + + // Supported sizes check + size_t pooling_dims = X_shape.NumDimensions() - 2; + if (pooling_dims > 3) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported pooling size."); + } + + // Get pooled index tensor + const Tensor* I = context->Input(1); + const TensorShape& I_shape = I->Shape(); + const int64_t* I_data = I->template Data(); + + ORT_RETURN_IF_NOT(I_shape == X_shape, "Index tensor shape should be same as that of the input data tensor to unpool."); + + // Calculate output tensor shape from attributes + std::vector inferredOutputShape(X_shape.NumDimensions()); + + // Copy batch and channel dims + inferredOutputShape[0] = X_shape[0]; + inferredOutputShape[1] = X_shape[1]; + + // For feature dims calculate reversing the formula used for Maxpool + for (auto dim = 0; dim < kernel_shape_.size(); ++dim) { + inferredOutputShape[dim + 2] = (X_shape[dim + 2] - 1) * strides_[dim] - (pads_[dim + 2] + pads_[kernel_shape_.size() + dim + 4]) + kernel_shape_[dim]; + } + + // If outputshape is provided use that to infer additional padding. + std::vector inferredPads; + std::vector givenOutputShape; + bool padsInferred = false; + + if (num_inputs_ == 3) { + auto& tensor_shape = *context->Input(2); + ORT_RETURN_IF_NOT(tensor_shape.Shape().GetDims().size() == 1, "Shape must be 1 dimensional as it's tensor data is a shape"); + + // Turn the shape tensor data into an actual shape + const int64_t* p_shape = tensor_shape.template Data(); + std::vector shape{p_shape, p_shape + tensor_shape.Shape().Size()}; + givenOutputShape = shape; + + inferredPads.resize(inferredOutputShape.size() * 2, 0); + + // calculate if output shape has any padding over the inferred shape for feature dims. + for (auto dim = 2; dim < shape.size(); dim++) { + ORT_RETURN_IF_NOT(inferredOutputShape[dim] <= shape[dim], "Incorrect output shape"); + + int64_t inferredPad = shape[dim] - inferredOutputShape[dim]; + ORT_RETURN_IF_NOT(inferredPad <= kernel_shape_[dim - 2], "Incorrect output shape"); + + if (inferredPad > 0) { + padsInferred = true; + if (inferredPad == kernel_shape_[dim - 2]) { + inferredPads[dim] = 1; + inferredPads[dim + inferredOutputShape.size()] = inferredPad - 1; + } else { + inferredPads[dim + inferredOutputShape.size()] = inferredPad; + } + } + } + } + + // unpool + int64_t totalPooledElem = 1; + int64_t totalOutputElem = 1; + + for (auto dim = 0; dim < X_shape.NumDimensions(); dim++) { + totalPooledElem *= X_shape[dim]; + totalOutputElem *= inferredOutputShape[dim]; + } + + // if there are no pads inferred from outputshape simply create the new unpooled tensor + if (!padsInferred) { + TensorShape shape(inferredOutputShape); + + Tensor* Y = context->Output(0, shape); + auto Y_data = Y->template MutableData(); + auto out = gsl::make_span(Y_data, Y->Shape().Size()); + std::fill_n(out.data(), out.size(), 0.f); + + for (auto curElem = 0; curElem < totalPooledElem; ++curElem) { + out[I_data[curElem]] = X_data[curElem]; + } + } else { + // If the output shape has pads over the inferred dims , first + // create the tensor with the inferred dims and add the padding. + + // Generate tensor with inferred dims. + TensorShape shape(inferredOutputShape); + + AllocatorPtr alloc; + ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + auto element_type = DataTypeImpl::GetType(); + + void* buffer = alloc->Alloc(sizeof(float) * shape.Size()); + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + alloc->Info(), + alloc); + + float* p = p_tensor->template MutableData(); + + auto out = gsl::make_span(p, p_tensor->Shape().Size()); + std::fill_n(out.data(), out.size(), 0.f); + + for (auto curElem = 0; curElem < totalPooledElem; ++curElem) { + out[I_data[curElem]] = X_data[curElem]; + } + + std::vector output_dims(inferredOutputShape); + size_t dimension_count = output_dims.size(); + + std::vector input_starts; + std::vector input_extents; + + // Calculate output dimensions + for (size_t i = 0; i < dimension_count; i++) { + input_starts.push_back(slices_[i]); + input_extents.push_back(output_dims[i] + slices_[i] + slices_[i + dimension_count]); + output_dims[i] += inferredPads[i] + inferredPads[i + dimension_count] + slices_[i] + slices_[i + dimension_count]; + } + + // setup output object + TensorShape output_shape(givenOutputShape); + Tensor* Y = context->Output(0, output_shape); + auto Y_data = Y->template MutableData(); + + auto outData = gsl::make_span(Y_data, Y->Shape().Size()); + + std::fill_n(outData.data(), outData.size(), 0.f); + + // add padding + TensorPitches output_pitches(*Y); + size_t alignSkip = 0; // Amount to skip to align to where the next input tensor data needs to be written + + // Initial skip, sum up the begin padding on each axis + for (size_t i = 0; i < dimension_count; i++) + alignSkip += inferredPads[i] * output_pitches[i]; + + size_t inner_axis = dimension_count - 1; + + TensorAxisCounters input_counters(*p_tensor); + SliceIterator input(*p_tensor, input_starts, input_extents); + + while (input_counters) { + Y_data += alignSkip; + { + Y_data = input.CopyInnermostAxis(Y_data); + int64_t prePad = inferredPads[inner_axis]; + int64_t postPad = inferredPads[inner_axis + dimension_count]; + Y_data += postPad; + alignSkip = prePad; + } + // Calculate the size of the next block of padding (skipping over the innermost axis since that's already done) + while (input_counters.Increment()) { + ptrdiff_t inner_pitch = output_pitches[input_counters.Axis()]; + int64_t prePad = inferredPads[input_counters.Axis()]; + int64_t postPad = inferredPads[input_counters.Axis() + dimension_count]; + Y_data += inner_pitch * postPad; + alignSkip += inner_pitch * prePad; + } + } + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm.h b/onnxruntime/core/providers/cpu/nn/batch_norm.h index f5e66849d6a26..66601c9c2e26d 100644 --- a/onnxruntime/core/providers/cpu/nn/batch_norm.h +++ b/onnxruntime/core/providers/cpu/nn/batch_norm.h @@ -27,7 +27,7 @@ namespace onnxruntime { template -class BatchNorm final : public OpKernel { +class BatchNorm : public OpKernel { public: BatchNorm(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { float tmp_eplison; @@ -38,7 +38,7 @@ class BatchNorm final : public OpKernel { Status Compute(OpKernelContext* p_op_kernel_context) const override; - private: + protected: float epsilon_ = 1e-5f; int64_t is_test_; // ignored in this implementation since we're doing inferencing only. }; diff --git a/onnxruntime/core/providers/cpu/nn/unpool.h b/onnxruntime/core/providers/cpu/nn/unpool.h new file mode 100644 index 0000000000000..012c1da30d140 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/unpool.h @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/autopad_type.h" + +namespace onnxruntime { + +class MaxUnpool : public OpKernel { + public: + MaxUnpool(const OpKernelInfo& info) : OpKernel(info) { + ORT_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape_).IsOK(), + "No kernel shape is set."); + + num_inputs_ = OpKernel::Node().InputDefs().size(); + + if (num_inputs_ == 3 && !pads_.empty()) { + // ignore pads attribute value + } + + // setup defaults. + if (!info.GetAttrs("pads", pads_).IsOK() || pads_.empty()) { + pads_.resize(kernel_shape_.size() * 2, 0); + } + + if (!info.GetAttrs("strides", strides_).IsOK() || strides_.empty()) { + strides_.resize(kernel_shape_.size(), 1); + } + + for (size_t dim = 0; dim < kernel_shape_.size(); ++dim) { + ORT_ENFORCE(kernel_shape_[dim] > 0); + ORT_ENFORCE(pads_[dim] < kernel_shape_[dim] && pads_[dim + kernel_shape_.size()] < kernel_shape_[dim], + "Pad should be smaller than kernel."); + } + + ORT_ENFORCE(strides_.size() == kernel_shape_.size()); + + // Add 4 pad values (0) for batch and channel dimensions + pads_.insert(pads_.begin(), {0, 0}); + pads_.insert(pads_.begin() + 2 + kernel_shape_.size(), {0, 0}); + + // Separate out any negative pads_ into the slices_ array + slices_.resize(pads_.size(), 0); + for (size_t index = 0; index < pads_.size(); index++) { + if (pads_[index] < 0) { + slices_[index] = pads_[index]; + pads_[index] = 0; + } + } + } + + ~MaxUnpool() override{}; + + Status Compute(OpKernelContext* context) const override; + + private: + std::vector kernel_shape_; + std::vector pads_; + std::vector strides_; + std::vector slices_; // All of the negative padding values are separated out into slices_ + int64_t num_inputs_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index f60ba073a64bf..c7955f9b2beab 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -17,9 +17,9 @@ OrtCreateDefaultAllocator OrtCreateInferenceSession OrtCreateRunOptions OrtCreateSessionOptions -OrtCreateTensorAsONNXValue +OrtCreateTensorAsOrtValue OrtCreateTensorTypeAndShapeInfo -OrtCreateTensorWithDataAsONNXValue +OrtCreateTensorWithDataAsOrtValue OrtDisableCpuMemArena OrtDisableMemPattern OrtDisableProfiling @@ -50,7 +50,12 @@ OrtInferenceSessionGetOutputTypeInfo OrtInitialize OrtInitializeWithCustomLogger OrtIsTensor +OrtReleaseAllocatorInfo +OrtReleaseEnv OrtReleaseObject +OrtReleaseSession +OrtReleaseStatus +OrtReleaseValue OrtRunInference OrtRunOptionsGetRunLogVerbosityLevel OrtRunOptionsGetRunTag @@ -63,9 +68,4 @@ OrtSetSessionLogId OrtSetSessionLogVerbosityLevel OrtSetSessionThreadPoolSize OrtSetTensorElementType -OrtTensorProtoToONNXValue -ReleaseONNXEnv -ReleaseOrtAllocatorInfo -ReleaseONNXSession -ReleaseONNXStatus -ReleaseONNXValue +OrtTensorProtoToOrtValue diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.h b/onnxruntime/core/providers/cpu/tensor/cast_op.h index e75c08790cb9e..54feb86a0825a 100644 --- a/onnxruntime/core/providers/cpu/tensor/cast_op.h +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.h @@ -5,9 +5,9 @@ #include "core/common/common.h" #include "core/framework/op_kernel.h" +#include "core/util/math.h" #include "core/util/math_cpuonly.h" #include "Eigen/src/Core/arch/CUDA/Half.h" -#include "core/util/math_cpuonly.h" #if defined(USE_MLAS) && defined(_M_AMD64) #include "core/mlas/inc/mlas.h" diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index c9030b27c64ff..fb419ac9221d0 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -15,7 +15,7 @@ struct CUDAProviderFactory { CUDAProviderFactory(); }; -ONNXStatus* ORT_API_CALL CreateCuda(void* this_, OrtProvider** out) { +OrtStatus* ORT_API_CALL CreateCuda(void* this_, OrtProvider** out) { CUDAExecutionProviderInfo info; CUDAProviderFactory* this_ptr = (CUDAProviderFactory*)this_; info.device_id = this_ptr->device_id; diff --git a/onnxruntime/core/providers/mkldnn/activation/activations.cc b/onnxruntime/core/providers/mkldnn/activation/activations.cc new file mode 100644 index 0000000000000..e43d7aa3934a3 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/activation/activations.cc @@ -0,0 +1,204 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/activation/activations.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN Pool primitive. +struct ReluParams { + mkldnn::memory::dims& src_dims; + mkldnn::memory::dims& dst_dims; + size_t num_dimensions; + + ReluParams(mkldnn::memory::dims& src_dims, mkldnn::memory::dims& dst_dims, + size_t dimensions = 0) + : src_dims(src_dims), + dst_dims(dst_dims), + num_dimensions(dimensions) {} + + // Used as the key for Pool Primitive Reuse Pool. + std::string ToString() const { + std::string key; + key.reserve(64); + key.append("Relu_"); + AddDimsToKey(key, src_dims); + AddDimsToKey(key, dst_dims); + return key; + } +}; + +template +class ReluPrimitive final : public PrimitiveBase { + public: + explicit ReluPrimitive(const ReluParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.relu_fwd == nullptr) { + Initialize(params); + } + } + + ~ReluPrimitive() = default; + + void Compute(const T* src_data, const T* dst_data) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + context_.stream->submit(context_.net); + + context_.src_mem->set_data_handle(nullptr); + context_.dst_mem->set_data_handle(nullptr); + return; + } + + std::unique_ptr + GetDstMemoryDesc() const { return context_.dst_md; } + + std::unique_ptr + GetPrimitiveDesc() const { + return context_.relu_fwd_pd; + } + + private: + struct ReluContext { + mkldnn::memory::format src_fmt; + + std::unique_ptr src_mem; + std::unique_ptr dst_mem; + + size_t src_size; + size_t dst_size; + + std::unique_ptr fwd_desc; + std::unique_ptr relu_fwd_pd; + std::unique_ptr relu_fwd; + + std::unique_ptr src_md; + std::unique_ptr dst_md; + + std::unique_ptr stream; + std::vector net; + }; + + void Initialize(const ReluParams& params) { + + mkldnn::memory::format fmt = mkldnn::memory::format::any; + switch (params.num_dimensions) { + case 1: { fmt = mkldnn::memory::format::x; break; } + case 2: { fmt = mkldnn::memory::format::nc; break; } + case 3: { fmt = mkldnn::memory::format::ntc; break; } + case 4: { fmt = mkldnn::memory::format::nchw; break; } + case 5: { fmt = mkldnn::memory::format::ncdhw; break; } + default: { fmt = mkldnn::memory::format::any; break; } + } + + context_.src_md.reset(new mkldnn::memory::desc({ params.src_dims}, MklDnnType(), fmt)); + + mkldnn::algorithm algo = mkldnn::algorithm::eltwise_relu; + context_.fwd_desc.reset(new mkldnn::eltwise_forward::desc( + mkldnn::prop_kind::forward_inference, algo, *context_.src_md, 0)); + + context_.relu_fwd_pd.reset(new mkldnn::eltwise_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + context_.src_fmt = static_cast( + context_.relu_fwd_pd.get()->src_primitive_desc().desc().data.format); + + context_.src_size = context_.relu_fwd_pd.get()->src_primitive_desc().get_size(); + context_.dst_size = context_.relu_fwd_pd.get()->dst_primitive_desc().get_size(); + + context_.src_mem.reset(new mkldnn::memory(context_.relu_fwd_pd.get()->src_primitive_desc(), nullptr)); + context_.dst_mem.reset(new mkldnn::memory(context_.relu_fwd_pd.get()->dst_primitive_desc(), nullptr)); + context_.relu_fwd.reset( + new mkldnn::eltwise_forward(*context_.relu_fwd_pd, *context_.src_mem, *context_.dst_mem)); + context_.net.push_back(*context_.relu_fwd); + } + + ReluContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Relu primitives which are expensive +// to instantiate. To address thread safety, the primitives are stored in a map +// on thread local storage. +template +class ReluPrimitivePool : public PrimitivePool { + public: + static ReluPrimitive* Get(const ReluParams& params) { + ReluPrimitive* primitive = dynamic_cast*>( + ReluPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto relu_primitive = std::make_unique>(params); + primitive = relu_primitive.get(); + ReluPrimitivePool::GetInstance().SetPrimitive(params.ToString(), + std::move(relu_primitive)); + } + return primitive; + } + + private: + ReluPrimitivePool() = default; + ~ReluPrimitivePool() = default; + + static ReluPrimitivePool& GetInstance() { + static ReluPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status Relu::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + + if (X->Shape().NumDimensions() > 5 ) { + return onnxruntime::Relu::Compute(context); + } + + const TensorShape& y_shape = Y->Shape(); + auto& y_dims = y_shape.GetDims(); + + const T* src_data = X->template Data(); + T* dst_data = Y->template MutableData(); + + mkldnn::memory::dims src_dims_mkl(x_dims.begin(), x_dims.end()); + mkldnn::memory::dims dst_dims_mkl(y_dims.begin(), y_dims.end()); + + try { + ReluParams pool_params(src_dims_mkl, dst_dims_mkl, x_shape.NumDimensions()); + ReluPrimitive* relulPrimitive = ReluPrimitivePool::Get(pool_params); + + relulPrimitive->Compute(src_data, dst_data); + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, + ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + Relu, + kOnnxDomain, + 6, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Relu); + +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/activation/activations.h b/onnxruntime/core/providers/mkldnn/activation/activations.h new file mode 100644 index 0000000000000..be8ba3537b91c --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/activation/activations.h @@ -0,0 +1,20 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/activation/activations.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class Relu : public onnxruntime::Relu { + public: + Relu(const OpKernelInfo& info) : onnxruntime::Relu(info) {} + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/math/sum.cc b/onnxruntime/core/providers/mkldnn/math/sum.cc new file mode 100644 index 0000000000000..e7d3970f44b98 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/math/sum.cc @@ -0,0 +1,231 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/math/sum.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN Sum primitives. +struct SumParams { + const std::vector& src_dims; + const mkldnn::memory::dims& dst_dim; + const int num_inputs; + const int num_dimensions; + + SumParams(const std::vector& dims, + const mkldnn::memory::dims& dst_dims, const int numinputs, + const int dimensions) + : src_dims(dims), + dst_dim(dst_dims), + num_inputs(numinputs), + num_dimensions(dimensions) {} + + // Used as the key for Sum Primitive Reuse Sum. + std::string ToString() const { + std::string key; + key.reserve(64); + key.append("sum_"); + for (size_t i = 0; i < src_dims.size(); i++) { + AddDimsToKey(key, src_dims[i]); + } + AddDimsToKey(key, dst_dim); + return key; + } +}; + +template +class SumPrimitive final : public PrimitiveBase { + public: + explicit SumPrimitive(const SumParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.sum_pd == nullptr) { + Initialize(params); + } + } + + ~SumPrimitive() = default; + + void Compute(OpKernelContext* context, int numinputs) { + const Tensor* X1 = context->Input(0); + Tensor* Y = context->Output(0, X1->Shape()); + T* dst_data = Y->template MutableData(); + + context_.dst_mem->set_data_handle( + static_cast(static_cast(dst_data))); + + for (int i = 0; i < numinputs; i++) { + const Tensor* X = context->Input(i); + const T* src_data = X->template Data(); + context_.srcs_memory[i].set_data_handle( + static_cast(const_cast(src_data))); + } + context_.stream->submit(context_.net); + + for (int i = 0; i < numinputs; i++) { + context_.srcs_memory[i].set_data_handle(nullptr); + } + + } + + std::unique_ptr GetDstMemoryDesc() const { + return context_.dst_md; + } + + std::unique_ptr + GetPrimitiveDesc() const { + return context_.sum_pd; + } + + private: + struct SumContext { + std::unique_ptr src_md; + std::unique_ptr dst_md; + + std::vector srcs_memory; + std::unique_ptr dst_mem; + + std::vector srcs_pd; + std::unique_ptr src_mpd; + std::unique_ptr dst_pd; + std::unique_ptr sum_pd; + + std::unique_ptr stream; + std::vector net; + }; + + void Initialize(const SumParams& params) { + std::vector coeff; + + mkldnn::memory::format fmt = mkldnn::memory::format::any; + switch (params.num_dimensions) { + case 1: { fmt = mkldnn::memory::format::x; break; } + case 2: { fmt = mkldnn::memory::format::nc; break; } + case 3: { fmt = mkldnn::memory::format::ntc; break; } + case 4: { fmt = mkldnn::memory::format::nchw; break; } + case 5: { fmt = mkldnn::memory::format::ncdhw; break; } + default: { fmt = mkldnn::memory::format::any; break; } + } + + for (int i = 0; i < params.num_inputs; i++) { + context_.src_md.reset( + new mkldnn::memory::desc({params.src_dims[i]}, MklDnnType(), fmt)); + auto mpd = mkldnn::memory::primitive_desc(*context_.src_md, cpu_engine_); + auto src_memory = mkldnn::memory(mpd, nullptr); + + context_.srcs_pd.push_back(mpd); + context_.srcs_memory.push_back(src_memory); + coeff.push_back(1.0); + } + + std::unique_ptr dst; + context_.dst_md.reset(new mkldnn::memory::desc( + {params.dst_dim}, MklDnnType(), mkldnn::memory::format::any)); + context_.sum_pd.reset(new mkldnn::sum::primitive_desc( + *context_.dst_md, coeff, context_.srcs_pd)); + context_.dst_mem.reset(new mkldnn::memory( + context_.sum_pd->dst_primitive_desc(), nullptr)); + + std::vector inputs; + for (int i = 0; i < params.num_inputs; i++) { + inputs.push_back(context_.srcs_memory[i]); + } + auto c = mkldnn::sum(*context_.sum_pd, inputs, *context_.dst_mem); + context_.net.push_back(c); + } + + SumContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Sum primitives which are +// expensive to instantiate. To address thread safety, the primitives +// are stored in a map on thread local storage. + +template +class SumPrimitivePool : public PrimitivePool { + public: + static SumPrimitive* Get(const SumParams& params) { + SumPrimitive* primitive = dynamic_cast*>( + SumPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto sum_primitive = std::make_unique>(params); + primitive = sum_primitive.get(); + SumPrimitivePool::GetInstance().SetPrimitive( + params.ToString(), std::move(sum_primitive)); + } + return primitive; + } + + private: + SumPrimitivePool() = default; + ~SumPrimitivePool() = default; + + static SumPrimitivePool& GetInstance() { + static SumPrimitivePool pool; + return pool; + } +}; +} // namespace_ + +template +Status Sum::Compute(OpKernelContext* context) const { + int num_inputs = static_cast(OpKernel::Node().InputDefs().size()); + + ORT_ENFORCE(num_inputs > 0, "MKLDNN Sum kernel: Must have at least one input"); + + if (num_inputs == 1) { + return onnxruntime::Sum_6::Compute(context); + } + + std::vector src_dims; + + const Tensor* X1 = context->Input(0); + Tensor* Y = context->Output(0, X1->Shape()); + int dimensions = static_cast(X1->Shape().NumDimensions()); + + const TensorShape& x_shape = X1->Shape(); + const auto& x_dims = x_shape.GetDims(); + mkldnn::memory::dims src_dim(x_dims.begin(), x_dims.end()); + + mkldnn::memory::dims dst_dims_mkl( + Y->Shape().GetDims().begin(), Y->Shape().GetDims().end()); + + for (int i = 0; i < num_inputs; i++) { + const Tensor* X = context->Input(i); + mkldnn::memory::dims src_dims_mkl( + X->Shape().GetDims().begin(), X->Shape().GetDims().end()); + src_dims.push_back(src_dims_mkl); + } + try { + SumParams parameters(src_dims, dst_dims_mkl, num_inputs, dimensions); + SumPrimitive* sum_primitive = SumPrimitivePool::Get(parameters); + ORT_RETURN_IF_NOT(sum_primitive != nullptr); + sum_primitive->Compute(context, num_inputs); + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, + ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + Sum, + kOnnxDomain, + 6, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sum); + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/math/sum.h b/onnxruntime/core/providers/mkldnn/math/sum.h new file mode 100644 index 0000000000000..79d5326c6d70f --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/math/sum.h @@ -0,0 +1,22 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "../mkldnn_execution_provider.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class Sum final : public onnxruntime::Sum_6 { + public: + Sum(const OpKernelInfo& info) : onnxruntime::Sum_6(info) {} + + Status Compute(OpKernelContext* context) const override; + +private: +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/memcpy_s.h b/onnxruntime/core/providers/mkldnn/memcpy_s.h new file mode 100644 index 0000000000000..3db97157118fb --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/memcpy_s.h @@ -0,0 +1,13 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once + +#ifdef _WIN32 +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +// memcpy is deprecated. Replacing it with more secure equivalent memcpy_s +// +#define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, MIN(destsz, srcsz)) + diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc index d8889419a7251..299fcfafbbae0 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc @@ -65,6 +65,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, Gemm); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, MemcpyFromHost); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, MemcpyToHost); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 6, Relu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 6, Sum); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, BatchNormalization); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, 8, float, AveragePool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 8, float, GlobalAveragePool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool); @@ -77,6 +80,9 @@ void RegisterMKLDNNKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index e5820f8951514..911dd2a1eb22f 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -4,10 +4,12 @@ #pragma once #include +#include #include "core/framework/allocatormgr.h" #include "core/framework/execution_provider.h" #include "core/graph/graph_transformer.h" +#include "mkldnn.hpp" namespace onnxruntime { @@ -37,6 +39,23 @@ class MKLDNNExecutionProvider : public IExecutionProvider { } virtual std::shared_ptr GetKernelRegistry() const override; + + std::shared_ptr GetWeightMemory(std::string weightName) { + + auto iter = weights_mem_map.find(weightName); + if (iter != weights_mem_map.end()) + return iter->second; + return nullptr; + } + + std::map>& GetWeightsMap() { + return weights_mem_map; + } + +private: + // mkldnn formatted weights(filer data) memory from first iteration + // saved by weights name + std::map> weights_mem_map; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc b/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc index d9d9a543462b6..f9a7b0f063b94 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc +++ b/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc @@ -15,7 +15,7 @@ struct MkldnnProviderFactory { MkldnnProviderFactory(); }; -ONNXStatus* ORT_API_CALL CreateMkldnn(void* this_, OrtProvider** out) { +OrtStatus* ORT_API_CALL CreateMkldnn(void* this_, OrtProvider** out) { MKLDNNExecutionProviderInfo info; MkldnnProviderFactory* this_ptr = (MkldnnProviderFactory*)this_; info.create_arena = this_ptr->create_arena; diff --git a/onnxruntime/core/providers/mkldnn/nn/batch_norm.cc b/onnxruntime/core/providers/mkldnn/nn/batch_norm.cc new file mode 100644 index 0000000000000..b908415023f3f --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/batch_norm.cc @@ -0,0 +1,287 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/nn/batch_norm.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/mkldnn/memcpy_s.h" +#include "core/providers/cpu/nn/batch_norm_helper.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN BatchNorm primitive. +struct BatchNormParams { + const mkldnn::memory::dims& src_dims; + const mkldnn::memory::dims& scale_dims; + const mkldnn::memory::dims& b_dims; + const mkldnn::memory::dims& mean_dims; + const mkldnn::memory::dims& var_dims; + const mkldnn::memory::dims& dst_dims; + const float epsilon; + const int num_dimensions; + + BatchNormParams(const mkldnn::memory::dims& src_dims_mkl, + const mkldnn::memory::dims& scale_dims_mkl, + const mkldnn::memory::dims& b_dims_mkl, const mkldnn::memory::dims& mean_dims_mkl, + const mkldnn::memory::dims& var_dims_mkl, const mkldnn::memory::dims& dst_dims_mkl, + const float eps, const int dimensions) + : src_dims(src_dims_mkl), + scale_dims(scale_dims_mkl), + b_dims(b_dims_mkl), + mean_dims(mean_dims_mkl), + var_dims(var_dims_mkl), + dst_dims(dst_dims_mkl), + epsilon(eps), + num_dimensions(dimensions) {} + + // Used as the key for BatchNorm Primitive Reuse Pool. + std::string ToString() const { + std::string key; + key.reserve(128); + key.append("BatchNorm_"); + AddDimsToKey(key, src_dims); + AddDimsToKey(key, scale_dims); + AddDimsToKey(key, b_dims); + AddDimsToKey(key, mean_dims); + AddDimsToKey(key, var_dims); + AddDimsToKey(key, dst_dims); + return key; + } +}; + +template +class BatchNormPrimitive final : public PrimitiveBase { + public: + explicit BatchNormPrimitive(const BatchNormParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.batchnorm_fwd == nullptr) { + Initialize(params); + } + } + + ~BatchNormPrimitive() = default; + + void Compute(const T* src_data, const T* scale_data, const T* b_data, + const T* mean_data, const T* var_data, const T* dst_data, + int scale_dims_channels) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.mean_mem->set_data_handle( + static_cast(const_cast(mean_data))); + context_.var_mem->set_data_handle( + static_cast(const_cast(var_data))); + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + + T* scaleShift_buf = static_cast(context_.scale_shift_mem->get_data_handle()); + + size_t src_bytes = sizeof(T) * scale_dims_channels; + size_t dst_bytes = sizeof(T) * scale_dims_channels; + + MEMCPY_S(scaleShift_buf, scale_data, src_bytes, dst_bytes); + MEMCPY_S(&scaleShift_buf[scale_dims_channels], b_data, src_bytes, dst_bytes); + context_.stream->submit(context_.net); + return; + } + + std::unique_ptr + GetPrimitiveDesc() const { + return context_.conv_fwd_pd; + } + + private: + struct BatchNormContext { + std::unique_ptr src_mem; + std::unique_ptr scale_shift_mem; + std::unique_ptr mean_mem; + std::unique_ptr var_mem; + std::unique_ptr dst_mem; + + std::unique_ptr src_md; + std::unique_ptr scale_shift_md; + std::unique_ptr mean_md; + std::unique_ptr var_md; + std::unique_ptr dst_md; + + std::unique_ptr batchnorm_fwd; + std::unique_ptr + batchnorm_fwd_pd; + + std::unique_ptr stream; + std::vector net; + }; + + void Initialize(const BatchNormParams& params) { + mkldnn::memory::format fmt = mkldnn::memory::format::any; + switch (params.num_dimensions) { + case 1: { fmt = mkldnn::memory::format::x; break; } + case 2: { fmt = mkldnn::memory::format::nc; break; } + case 3: { fmt = mkldnn::memory::format::ntc; break; } + case 4: { fmt = mkldnn::memory::format::nchw; break; } + case 5: { fmt = mkldnn::memory::format::ncdhw; break; } + default: { fmt = mkldnn::memory::format::any; break; } + } + context_.src_md.reset(new mkldnn::memory::desc( + { params.src_dims }, MklDnnType(), fmt)); + + context_.scale_shift_md.reset(new mkldnn::memory::desc( + { 2, params.scale_dims[0] }, MklDnnType(), mkldnn::memory::format::nc)); + + context_.mean_md.reset(new mkldnn::memory::desc( + { params.mean_dims }, MklDnnType(), mkldnn::memory::format::x)); + context_.var_md.reset(new mkldnn::memory::desc( + { params.var_dims }, MklDnnType(), mkldnn::memory::format::x)); + context_.dst_md.reset(new mkldnn::memory::desc( + { params.dst_dims }, MklDnnType(), fmt)); + + context_.src_mem.reset( + new mkldnn::memory({ *context_.src_md, cpu_engine_ }, nullptr)); + + // scale_shift_mem will allocate 2*C*sizeof(float) buffer + // + context_.scale_shift_mem.reset( + new mkldnn::memory({ *context_.scale_shift_md, cpu_engine_ })); + + context_.mean_mem.reset( + new mkldnn::memory({ *context_.mean_md, cpu_engine_ }, nullptr)); + context_.var_mem.reset( + new mkldnn::memory({ *context_.var_md, cpu_engine_ }, nullptr)); + + context_.batchnorm_fwd.reset(new mkldnn::batch_normalization_forward::desc( + mkldnn::prop_kind::forward_inference, *context_.src_md, params.epsilon, + mkldnn::batch_normalization_flag::use_scale_shift | + mkldnn::batch_normalization_flag::use_global_stats)); + + context_.batchnorm_fwd_pd.reset( + new mkldnn::batch_normalization_forward::primitive_desc( + *context_.batchnorm_fwd, cpu_engine_)); + + context_.dst_mem.reset( + new mkldnn::memory( + context_.batchnorm_fwd_pd->dst_primitive_desc(), nullptr)); + + auto bn = mkldnn::batch_normalization_forward( + *context_.batchnorm_fwd_pd, + (const mkldnn::primitive::at)*context_.src_mem, + (const mkldnn::primitive::at)*context_.mean_mem, + (const mkldnn::primitive::at)*context_.var_mem, + (const mkldnn::memory)*context_.scale_shift_mem, + (const mkldnn::memory) *context_.dst_mem); + + context_.net.push_back(bn); + } + + BatchNormContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN BatchNorm primitives which are +// expensive to instantiate. To address thread safety, the primitives are +// stored in a map on thread local storage. +template +class BatchNormPrimitivePool : public PrimitivePool { + public: + static BatchNormPrimitive* Get(const BatchNormParams& params) { + BatchNormPrimitive* primitive = + dynamic_cast*>( + BatchNormPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto BatchNorm_primitive = std::make_unique>(params); + primitive = BatchNorm_primitive.get(); + BatchNormPrimitivePool::GetInstance().SetPrimitive( + params.ToString(), std::move(BatchNorm_primitive)); + } + return primitive; + } + + private: + BatchNormPrimitivePool() = default; + ~BatchNormPrimitivePool() = default; + + static BatchNormPrimitivePool& GetInstance() { + static BatchNormPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status BatchNorm::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + + int num_dimensions = static_cast(X->Shape().NumDimensions()); + if (num_dimensions == 3) { + // Fall back CPU implementation + return onnxruntime::BatchNorm::Compute(context); + } + + const T* src_data = X->template Data(); + + const Tensor* scale = context->Input(1); + const T* scale_data = scale->template Data(); + + const Tensor* B = context->Input(2); + const T* b_data = B->template Data(); + + const Tensor* mean = context->Input(3); + const T* mean_data = mean->template Data(); + + const Tensor* var = context->Input(4); + const T* var_data = var->template Data(); + + Tensor* Y = context->Output(0, X->Shape()); + T* dst_data = Y->template MutableData(); + + ORT_RETURN_IF_ERROR( + BatchNormHelper::ValidateInputs(X, scale, B, mean, var)); + + mkldnn::memory::dims src_dims_mkl( + X->Shape().GetDims().begin(), X->Shape().GetDims().end()); + mkldnn::memory::dims scale_dims_mkl( + scale->Shape().GetDims().begin(), scale->Shape().GetDims().end()); + mkldnn::memory::dims b_dims_mkl( + B->Shape().GetDims().begin(), B->Shape().GetDims().end()); + mkldnn::memory::dims mean_dims_mkl( + mean->Shape().GetDims().begin(), mean->Shape().GetDims().end()); + mkldnn::memory::dims var_dims_mkl( + var->Shape().GetDims().begin(), var->Shape().GetDims().end()); + + mkldnn::memory::dims dst_dims_mkl( + Y->Shape().GetDims().begin(), Y->Shape().GetDims().end()); + + try { + BatchNormParams batchNorm_params(src_dims_mkl, scale_dims_mkl, + b_dims_mkl, mean_dims_mkl, var_dims_mkl, dst_dims_mkl, + onnxruntime::BatchNorm::epsilon_, num_dimensions); + BatchNormPrimitive* batchNorm_primitive = + BatchNormPrimitivePool::Get(batchNorm_params); + ORT_RETURN_IF_NOT(batchNorm_primitive != nullptr); + batchNorm_primitive->Compute(src_data, scale_data, b_data, + mean_data, var_data, dst_data, scale_dims_mkl[0]); + + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, FAIL, "Status: ", e.status, ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + BatchNormalization, + kOnnxDomain, + 7, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + BatchNorm); + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/batch_norm.h b/onnxruntime/core/providers/mkldnn/nn/batch_norm.h new file mode 100644 index 0000000000000..bd4ddd23fae42 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/batch_norm.h @@ -0,0 +1,18 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/batch_norm.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class BatchNorm final : public onnxruntime::BatchNorm { + public: + BatchNorm(const OpKernelInfo& info) : onnxruntime::BatchNorm(info) {} + Status Compute(OpKernelContext* context) const override; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 355562bf5adcd..7fe0152d42e9f 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -352,6 +352,7 @@ Status Conv::Compute(OpKernelContext* context) const { dst_dims_mkl, strides_mkl, dilations_mkl, padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); + auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -402,20 +403,32 @@ Status Conv::Compute(OpKernelContext* context) const { src_data = static_cast(dst.get_data_handle()); } - // Reorder filter memory layout if necessary. - if (filter_format != conv_primitive->GetFilterMemoryFormat()) { - auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc(filter_dims_mkl, - MklDnnType(), - filter_format), - cpu_engine); - mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); - // allocate the size queried from memory primitive desc. it may not match tensor logical size due to - // mkldnn using padding to allow use of blocked format. - filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); - mkldnn::memory dst = mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get()); - MemoryReorderParams params(src, dst); - DoReorder(params); - filter_data = static_cast(dst.get_data_handle()); + // Reorder filter memory layout if necessary + // Avoid data reordering. Save filter memory in mkldnn format from first iteration + // in execution provider mapped by weight name. + std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); + std::shared_ptr filter_dst_mem =nullptr; + filter_dst_mem = provider_->GetWeightMemory(weightKey); + + if (filter_dst_mem == nullptr) { + if (filter_format != conv_primitive->GetFilterMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc( + filter_dims_mkl, MklDnnType(), filter_format), cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); + filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); + filter_dst_mem.reset( + new mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get())); + MemoryReorderParams params(src, *filter_dst_mem); + DoReorder(params); + filter_data = static_cast(filter_dst_mem->get_data_handle()); + { + // make assignment threadsafe + std::lock_guard lock(mutex_); + provider_->GetWeightsMap()[weightKey] = filter_dst_mem; + } + } + } else { + filter_data = static_cast(filter_dst_mem->get_data_handle()); } // Allocate dst buffer if reorder is necessary diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 8189a7ca6ba10..8dde0c899f577 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -4,18 +4,24 @@ #pragma once #include "core/framework/op_kernel.h" #include "core/providers/cpu/nn/conv.h" +#include "../mkldnn_execution_provider.h" namespace onnxruntime { namespace mkl_dnn { + template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); } Status Compute(OpKernelContext* context) const override; private: + MKLDNNExecutionProvider * provider_; + mutable std::mutex mutex_; }; } // namespace mkl_dnn } // namespace onnxruntime diff --git a/onnxruntime/core/session/IOBinding.cc b/onnxruntime/core/session/IOBinding.cc index 67a33be805612..cd24146d71105 100644 --- a/onnxruntime/core/session/IOBinding.cc +++ b/onnxruntime/core/session/IOBinding.cc @@ -60,10 +60,9 @@ common::Status IOBinding::CopyOneInputAcrossDevices(const SessionState& session_ size_t index = node_info.index; auto& node = *node_info.p_node; const KernelCreateInfo* kci = node_info.kci; - const auto* node_input_mem_types = (kci != nullptr) ? &kci->kernel_def->InputMemoryType() : nullptr; // node may declare input_mem_type to be on CPU explicitly - bool node_input_on_cpu = node_input_mem_types && MemTypeOnCpuExplicitly(*node_input_mem_types, index); + bool node_input_on_cpu = kci && MemTypeOnCpuExplicitly(kci->kernel_def->InputMemoryType(index)); auto& required_provider_type = node_input_on_cpu ? onnxruntime::kCpuExecutionProvider : node.GetExecutionProviderType(); if (!orig_mlvalue.IsTensor()) { // copying not supported for non-tensor types diff --git a/onnxruntime/core/session/default_cpu_allocator_c_api.cc b/onnxruntime/core/session/default_cpu_allocator_c_api.cc index 4267a7ce83cbe..8957fbdf4fe63 100644 --- a/onnxruntime/core/session/default_cpu_allocator_c_api.cc +++ b/onnxruntime/core/session/default_cpu_allocator_c_api.cc @@ -44,7 +44,7 @@ OrtDefaultAllocator() : ref_count_(1) { } ~OrtDefaultAllocator() { assert(ref_count_ == 0); - ReleaseOrtAllocatorInfo(cpuAllocatorInfo); + OrtReleaseAllocatorInfo(cpuAllocatorInfo); } public: @@ -71,7 +71,7 @@ ORT_ALLOCATOR_IMPL_END #define API_IMPL_END \ } \ catch (std::exception & ex) { \ - return CreateONNXStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ + return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } OrtAllocatorInterface OrtDefaultAllocator::table_ = { diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 52703a8b2235d..f61cd61122367 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -14,6 +14,7 @@ #include "core/graph/graph_viewer.h" #include "core/graph/graph_transformer.h" #include "core/graph/graph_transformer_mgr.h" +#include "core/graph/graph_utils.h" #include "core/graph/model.h" #include "core/framework/allocatormgr.h" #include "core/framework/customregistry.h" @@ -247,6 +248,44 @@ class InferenceSession::Impl { return common::Status::OK(); } + static common::Status TransformGraph(onnxruntime::Graph& graph, + const onnxruntime::GraphTransformerManager& graph_transformer_mgr, + const ExecutionProviders& providers, + KernelRegistryManager& kernel_registry_manager, + const InsertCastTransformer& insert_cast_transformer) { + // The transformer order: + // 1. built-in graph rewriter + // 2. each execution provider's transformer + // 3. do node placement according to kernel definition + // 4. insert copy nodes + // 5. insert cast nodes. + + // first apply the default/system/basic graph to graph optimizations. + ORT_RETURN_IF_ERROR(graph_transformer_mgr.ApplyAll(graph)); + + auto kernels{kernel_registry_manager.GetAllKernelRegistries()}; + + // Do partitioning based on execution providers' capability. + GraphPartitioner partitioner(kernel_registry_manager, providers); + ORT_RETURN_IF_ERROR(partitioner.Partition(graph)); + + // Insert copy nodes. + for (auto& provider : providers) { + if (provider->Type() != onnxruntime::kCpuExecutionProvider && + provider->Type() != onnxruntime::kMklDnnExecutionProvider && + provider->Type() != onnxruntime::kNupharExecutionProvider) { + TransformerMemcpyImpl copy_impl(graph, provider->Type()); + copy_impl.ModifyGraph(kernel_registry_manager); + } + } + + // Insert cast node/s. + bool modified = false; + ORT_RETURN_IF_ERROR(insert_cast_transformer.Apply(graph, modified)); + + return common::Status::OK(); + } + // memory allocations for a subgraph that are owned by InferenceSession struct SubgraphMemory { std::unique_ptr session_state; @@ -277,10 +316,8 @@ class InferenceSession::Impl { SessionStateInitializer initializer{*subgraph, *subgraph_info.session_state, execution_providers_, kernel_registry_manager_, *session_logger_}; - ORT_RETURN_IF_ERROR( - initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, - node.ImplicitInputDefs(), - session_options_.enable_sequential_execution)); + ORT_RETURN_IF_ERROR(initializer.CreatePlan(node.ImplicitInputDefs(), + session_options_.enable_sequential_execution)); ORT_RETURN_IF_ERROR(initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), subgraph_info.weights_buffers)); @@ -347,9 +384,21 @@ class InferenceSession::Impl { SessionStateInitializer session_initializer{graph, session_state_, execution_providers_, kernel_registry_manager_, *session_logger_}; - ORT_RETURN_IF_ERROR(session_initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, - {}, session_options_.enable_sequential_execution)); + // apply any transformations to the main graph and any subgraphs + ORT_RETURN_IF_ERROR(TransformGraph(graph, graph_transformation_mgr_, + execution_providers_, kernel_registry_manager_, + insert_cast_transformer_)); + + ORT_RETURN_IF_ERROR(utils::ForAllMutableSubgraphs(graph, [this](Graph& subgraph) { + return TransformGraph(subgraph, graph_transformation_mgr_, + execution_providers_, kernel_registry_manager_, + insert_cast_transformer_); + })); + + // now that all the transforms are done, call Resolve on the main graph. this will recurse into the subgraphs. + ORT_RETURN_IF_ERROR(graph.Resolve()); + ORT_RETURN_IF_ERROR(session_initializer.CreatePlan({}, session_options_.enable_sequential_execution)); ORT_RETURN_IF_ERROR(session_initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), weights_buffers_)); diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index fe3daab56735a..627821cb09b6b 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -35,7 +35,7 @@ using onnxruntime::MLStatus; using onnxruntime::MLValue; using onnxruntime::OutputDefList; using onnxruntime::Tensor; -using onnxruntime::ToONNXStatus; +using onnxruntime::ToOrtStatus; using onnxruntime::common::Status; #define ORT_API_RETURN_IF_ERROR(expr) \ @@ -65,10 +65,10 @@ struct OrtEnv : public onnxruntime::ObjectBase { }; #define API_IMPL_BEGIN try { -#define API_IMPL_END \ - } \ - catch (std::exception & ex) { \ - return CreateONNXStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ +#define API_IMPL_END \ + } \ + catch (std::exception & ex) { \ + return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } #define TENSOR_READ_API_BEGIN \ @@ -113,7 +113,7 @@ ORT_API_STATUS_IMPL(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_fu Status status = Environment::Create(env); if (status.IsOK()) *out = new OrtEnv(env.release(), default_logging_manager.release()); - return ToONNXStatus(status); + return ToOrtStatus(status); API_IMPL_END } @@ -129,11 +129,11 @@ ORT_API_STATUS_IMPL(OrtInitialize, OrtLoggingLevel default_warning_level, Status status = Environment::Create(env); if (status.IsOK()) *out = new OrtEnv(env.release(), default_logging_manager.release()); - return ToONNXStatus(status); + return ToOrtStatus(status); API_IMPL_END } -ORT_API_STATUS_IMPL(OrtGetStringTensorDataLength, _In_ const ONNXValue* value, _Out_ size_t* out) { +ORT_API_STATUS_IMPL(OrtGetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* out) { TENSOR_READ_API_BEGIN const auto* src = tensor.Data(); int64_t len = tensor.Shape().Size(); @@ -144,17 +144,17 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorDataLength, _In_ const ONNXValue* value, _ } *out = ret; } else - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "shape is invalid"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "shape is invalid"); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtFillStringTensor, _In_ ONNXValue* value, _In_ const char* const* s, size_t s_len) { +ORT_API_STATUS_IMPL(OrtFillStringTensor, _In_ OrtValue* value, _In_ const char* const* s, size_t s_len) { TENSOR_READWRITE_API_BEGIN auto* dst = tensor->MutableData(); auto len = static_cast(tensor->Shape().Size()); if (s_len < len) { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "input array is too short"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "input array is too short"); } for (size_t i = 0; i != len; ++i) { //allocate and copy @@ -165,8 +165,8 @@ ORT_API_STATUS_IMPL(OrtFillStringTensor, _In_ ONNXValue* value, _In_ const char* } template -ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocatorInterface** allocator, - std::unique_ptr* out) { +OrtStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocatorInterface** allocator, + std::unique_ptr* out) { size_t elem_count = 1; std::vector shapes(shape_len); for (size_t i = 0; i != shape_len; ++i) { @@ -176,11 +176,11 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocator size_t size_to_allocate; if (!IAllocator::CalcMemSizeForArray(sizeof(T), elem_count, &size_to_allocate)) { - return CreateONNXStatus(ORT_FAIL, "not enough memory"); + return OrtCreateStatus(ORT_FAIL, "not enough memory"); } void* p_data = (*allocator)->Alloc(allocator, size_to_allocate); if (p_data == nullptr) - return CreateONNXStatus(ORT_FAIL, "size overflow"); + return OrtCreateStatus(ORT_FAIL, "size overflow"); *out = std::make_unique(DataTypeImpl::GetType(), onnxruntime::TensorShape(shapes), static_cast(p_data), @@ -194,8 +194,8 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocator * this function will create a copy of the allocator info */ template -ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAllocatorInfo* info, - void* p_data, size_t p_data_len, std::unique_ptr* out) { +OrtStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAllocatorInfo* info, + void* p_data, size_t p_data_len, std::unique_ptr* out) { size_t elem_count = 1; std::vector shapes(shape_len); for (size_t i = 0; i != shape_len; ++i) { @@ -205,12 +205,12 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAll size_t size_to_allocate; if (!IAllocator::CalcMemSizeForArray(sizeof(T), elem_count, &size_to_allocate)) { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "size overflow"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "size overflow"); } if (size_to_allocate > p_data_len) { std::ostringstream oss; oss << "not enough space: expected " << size_to_allocate << ", got " << p_data_len; - return CreateONNXStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); } *out = std::make_unique(DataTypeImpl::GetType(), onnxruntime::TensorShape(shapes), @@ -223,9 +223,9 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAll /** * this function will create a copy of the allocator info */ -ORT_API_STATUS_IMPL(OrtCreateTensorWithDataAsONNXValue, _In_ const OrtAllocatorInfo* info, +ORT_API_STATUS_IMPL(OrtCreateTensorWithDataAsOrtValue, _In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, - OrtTensorElementDataType type, _Out_ ONNXValue** out) { + ONNXTensorElementDataType type, _Out_ OrtValue** out) { API_IMPL_BEGIN std::unique_ptr tensor; switch (type) { @@ -274,21 +274,21 @@ ORT_API_STATUS_IMPL(OrtCreateTensorWithDataAsONNXValue, _In_ const OrtAllocatorI std::ostringstream oss; oss << "type " << type << " is not supported in this function"; std::string errmsg = oss.str(); - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); } } std::unique_ptr value = std::make_unique(); value->Init(tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtCreateTensorAsONNXValue, _Inout_ OrtAllocator* allocator, - _In_ const size_t* shape, size_t shape_len, OrtTensorElementDataType type, - _Out_ ONNXValue** out) { +ORT_API_STATUS_IMPL(OrtCreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const size_t* shape, size_t shape_len, ONNXTensorElementDataType type, + _Out_ OrtValue** out) { API_IMPL_BEGIN std::unique_ptr tensor; switch (type) { @@ -338,34 +338,34 @@ ORT_API_STATUS_IMPL(OrtCreateTensorAsONNXValue, _Inout_ OrtAllocator* allocator, std::ostringstream oss; oss << "type " << type << " is not supported in this function"; std::string errmsg = oss.str(); - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); } } std::unique_ptr value = std::make_unique(); value->Init(tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } template -static ONNXStatus* CreateInferenceSessionImpl(_In_ OrtEnv* env, _In_ T model_path, - _In_ const OrtSessionOptions* options, - _Out_ ONNXSession** out) { +static OrtStatus* CreateInferenceSessionImpl(_In_ OrtEnv* env, _In_ T model_path, + _In_ const OrtSessionOptions* options, + _Out_ OrtSession** out) { API_IMPL_BEGIN auto sess = std::make_unique<::onnxruntime::InferenceSession>(options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager); Status status; if (options != nullptr && !options->custom_op_paths.empty()) { status = sess->LoadCustomOps(options->custom_op_paths); if (!status.IsOK()) - return ToONNXStatus(status); + return ToOrtStatus(status); } if (options != nullptr) for (OrtProviderFactoryInterface** p : options->provider_factories) { OrtProvider* provider; - ONNXStatus* error_code = (*p)->CreateProvider(p, &provider); + OrtStatus* error_code = (*p)->CreateProvider(p, &provider); if (error_code) return error_code; sess->RegisterExecutionProvider(std::unique_ptr( @@ -373,35 +373,35 @@ static ONNXStatus* CreateInferenceSessionImpl(_In_ OrtEnv* env, _In_ T model_pat } status = sess->Load(model_path); if (!status.IsOK()) - return ToONNXStatus(status); + return ToOrtStatus(status); status = sess->Initialize(); if (!status.IsOK()) - return ToONNXStatus(status); - *out = reinterpret_cast(sess.release()); + return ToOrtStatus(status); + *out = reinterpret_cast(sess.release()); return nullptr; API_IMPL_END } #ifdef _WIN32 ORT_API_STATUS_IMPL(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const wchar_t* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out) { + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out) { API_IMPL_BEGIN return CreateInferenceSessionImpl(env, model_path, options, out); API_IMPL_END } #else ORT_API_STATUS_IMPL(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const char* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out) { + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out) { API_IMPL_BEGIN return CreateInferenceSessionImpl(env, model_path, options, out); API_IMPL_END } #endif -ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, +ORT_API_STATUS_IMPL(OrtRunInference, _In_ OrtSession* sess, _In_ OrtRunOptions* run_options, - _In_ const char* const* input_names, _In_ const ONNXValue* const* input, size_t input_len, - _In_ const char* const* output_names1, size_t output_names_len, _Out_ ONNXValue** output) { + _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, + _In_ const char* const* output_names1, size_t output_names_len, _Out_ OrtValue** output) { API_IMPL_BEGIN auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); ::onnxruntime::NameMLValMap in; @@ -410,7 +410,7 @@ ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, auto kvp = in.insert(std::make_pair(std::string(input_names[i]), *reinterpret_cast(input[i]))); if (!kvp.second) { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "duplicated input name"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "duplicated input name"); } ::onnxruntime::MLValue& value = kvp.first->second; if (value.Fence()) @@ -420,7 +420,7 @@ ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, std::vector output_names(output_names_len); for (size_t i = 0; i != output_names_len; ++i) { if (output_names1[i] == nullptr || output_names1[i][0] == '\0') { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "output name cannot be empty"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "output name cannot be empty"); } output_names[i] = output_names1[i]; } @@ -443,20 +443,20 @@ ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, } if (!status.IsOK()) - return ToONNXStatus(status); + return ToOrtStatus(status); for (size_t i = 0; i != output_names_len; ++i) { ::onnxruntime::MLValue& value = fetches[i]; if (value.Fence()) value.Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); if (output[i] == nullptr) { - output[i] = reinterpret_cast(new MLValue(value)); + output[i] = reinterpret_cast(new MLValue(value)); } } return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtGetTensorMutableData, _In_ ONNXValue* value, _Out_ void** output) { +ORT_API_STATUS_IMPL(OrtGetTensorMutableData, _In_ OrtValue* value, _Out_ void** output) { TENSOR_READWRITE_API_BEGIN //TODO: test if it's a string tensor *output = tensor->MutableDataRaw(); @@ -464,13 +464,13 @@ ORT_API_STATUS_IMPL(OrtGetTensorMutableData, _In_ ONNXValue* value, _Out_ void** API_IMPL_END } -ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const ONNXValue* value, +ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const OrtValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len) { TENSOR_READ_API_BEGIN const auto* input = tensor.Data(); auto len = static_cast(tensor.Shape().Size()); if (offsets_len < len) { - return CreateONNXStatus(ORT_FAIL, "space is not enough"); + return OrtCreateStatus(ORT_FAIL, "space is not enough"); } { size_t ret = 0; @@ -478,7 +478,7 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const ONNXValue* value, ret += input[i].size(); } if (s_len < ret) { - return CreateONNXStatus(ORT_FAIL, "space is not enough"); + return OrtCreateStatus(ORT_FAIL, "space is not enough"); } } size_t f = 0; @@ -493,75 +493,75 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const ONNXValue* value, API_IMPL_END } -ORT_API_STATUS_IMPL(OrtTensorProtoToONNXValue, _Inout_ OrtAllocator* allocator, - const void* input, int input_len, _Out_ ONNXValue** out) { +ORT_API_STATUS_IMPL(OrtTensorProtoToOrtValue, _Inout_ OrtAllocator* allocator, + const void* input, int input_len, _Out_ OrtValue** out) { API_IMPL_BEGIN std::shared_ptr allocator_ = std::make_shared(allocator); ::ONNX_NAMESPACE::TensorProto proto; if (!proto.ParseFromArray(input, input_len)) { - return CreateONNXStatus(ORT_FAIL, "parse input tensor proto failed"); + return OrtCreateStatus(ORT_FAIL, "parse input tensor proto failed"); } std::unique_ptr value = std::make_unique(); Status st = onnxruntime::utils::TensorProtoToMLValue(proto, allocator_, nullptr, 0, *value); if (!st.IsOK()) - return ToONNXStatus(st); - *out = reinterpret_cast(value.release()); + return ToOrtStatus(st); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } -#define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(INPUT_TYPE, REAL_TYPE) \ - ORT_API(void, Release##INPUT_TYPE, INPUT_TYPE* value) { \ - delete reinterpret_cast(value); \ +#define DEFINE_RELEASE_ORT_OBJECT_FUNCTION(INPUT_TYPE, REAL_TYPE) \ + ORT_API(void, OrtRelease##INPUT_TYPE, Ort##INPUT_TYPE* value) { \ + delete reinterpret_cast(value); \ } -#define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(INPUT_TYPE, REAL_TYPE) \ - ORT_API(void, Release##INPUT_TYPE, INPUT_TYPE* value) { \ - delete[] reinterpret_cast(value); \ +#define DEFINE_RELEASE_ORT_OBJECT_FUNCTION_FOR_ARRAY(INPUT_TYPE, REAL_TYPE) \ + ORT_API(void, OrtRelease##INPUT_TYPE, Ort##INPUT_TYPE* value) { \ + delete[] reinterpret_cast(value); \ } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputCount, _In_ const ONNXSession* sess, _Out_ size_t* out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelInputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); *out = p.second->size(); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputCount, _In_ const ONNXSession* sess, _Out_ size_t* out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelOutputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); *out = p.second->size(); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelInputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); if (p.second->size() <= index) - return CreateONNXStatus(ORT_FAIL, "out of index"); + return OrtCreateStatus(ORT_FAIL, "out of index"); const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); return OrtTypeInfo::FromDataTypeImpl(type_proto, out); API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelOutputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); if (p.second->size() <= index) - return CreateONNXStatus(ORT_FAIL, "out of index"); + return OrtCreateStatus(ORT_FAIL, "out of index"); const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); return OrtTypeInfo::FromDataTypeImpl(type_proto, out); API_IMPL_END @@ -574,23 +574,23 @@ static char* StrDup(const std::string& str, OrtAllocator* allocator) { return output_string; } -static ONNXStatus* GetInputOutputNameImpl(_In_ const ONNXSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, bool is_input, - _Out_ char** output) { +static OrtStatus* GetInputOutputNameImpl(_In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, bool is_input, + _Out_ char** output) { auto session = reinterpret_cast(sess); std::pair p = is_input ? session->GetModelInputs() : session->GetModelOutputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); if (p.second == nullptr) - return CreateONNXStatus(ORT_FAIL, "internal error"); + return OrtCreateStatus(ORT_FAIL, "internal error"); const InputDefList& defs = *p.second; if (index >= defs.size()) - return CreateONNXStatus(ORT_FAIL, "index out of range"); + return OrtCreateStatus(ORT_FAIL, "index out of range"); *output = StrDup(defs[index]->Name(), allocator); return nullptr; } -ORT_API(int, OrtIsTensor, _In_ const ONNXValue* value) { +ORT_API(int, OrtIsTensor, _In_ const OrtValue* value) { auto v = reinterpret_cast(value); return v->IsTensor() ? 1 : 0; } @@ -618,24 +618,24 @@ ORT_API(const struct OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAlloc } } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputName, _In_ const ONNXSession* sess, size_t index, +ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Out_ char** output) { API_IMPL_BEGIN return GetInputOutputNameImpl(sess, index, allocator, true, output); API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputName, _In_ const ONNXSession* sess, size_t index, +ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Out_ char** output) { API_IMPL_BEGIN return GetInputOutputNameImpl(sess, index, allocator, false, output); API_IMPL_END } -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXValue, MLValue) -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXSession, ::onnxruntime::InferenceSession) -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(ONNXStatus, char) +DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Value, MLValue) +DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Session, ::onnxruntime::InferenceSession) +DEFINE_RELEASE_ORT_OBJECT_FUNCTION_FOR_ARRAY(Status, char) -ORT_API(void, ReleaseONNXEnv, OrtEnv* env) { +ORT_API(void, OrtReleaseEnv, OrtEnv* env) { OrtReleaseObject(env); } diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h index 62db5e9fabe51..9e84a41d042b2 100644 --- a/onnxruntime/core/util/math.h +++ b/onnxruntime/core/util/math.h @@ -24,10 +24,16 @@ // still keep it simple, so all platforms would be able to support it fairly // easily. +#ifdef USE_MKLML_FOR_BLAS +// when USE_MKLML is defined, use MKLML cblas for GEMM +#include "mkl_cblas.h" +#define CBLAS_ENUM_DEFINED_H +#else // We include the cblas header here so that we can obtain the macros from cblas. extern "C" { #include "core/framework/cblas.h" } +#endif #include "core/common/common.h" #include "core/framework/data_types.h" @@ -70,7 +76,7 @@ void Not(const int N, const T* x, T* y, Provider* provider); template void Powx(const int N, const T* a, const T b, T* y, Provider* provider); -#define DECLARE_BINARY_OP_BINARY_RESULT(name) \ +#define DECLARE_BINARY_OP_BINARY_RESULT(name) \ template \ void name(const int N, const T* a, const T* b, bool* y, Provider* provider); \ template \ @@ -93,7 +99,7 @@ DECLARE_BINARY_OP_BINARY_RESULT(Xor); #undef DECLARE_BINARY_OP_BINARY_RESULT -#define DECLARE_BINARY_OP(name) \ +#define DECLARE_BINARY_OP(name) \ template \ void name(const int N, const T* a, const T* b, T* y, Provider* provider); \ template \ diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index b3b1fbcb70c4b..fc149c415ab25 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -57,7 +57,8 @@ namespace math { // will delegate the Caffe math functions that are BLAS-related to either the // CBLAS call or the Eigen implementation. //////////////////////////////////////////////////////////////////////////////// -#ifdef USE_EIGEN_FOR_BLAS +// when USE_MKLDNN and USE_MKLML are defined, use cblas APIs for MKLML +#if defined(USE_EIGEN_FOR_BLAS) && !defined(USE_MKLML_FOR_BLAS) // Caffe2 gemm provides a simpler interface to the gemm functions, with the // limitation that the data has to be contiguous in memory. @@ -332,10 +333,14 @@ void Gemm( float* C, CPUMathUtil* /*context*/, MLDataType /*math_type*/) { - int lda = (TransA == CblasNoTrans) ? K : M; - int ldb = (TransB == CblasNoTrans) ? N : K; - cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, - beta, C, N); + int lda = gsl::narrow_cast((TransA == CblasNoTrans) ? K : M); + int ldb = gsl::narrow_cast((TransB == CblasNoTrans) ? N : K); + cblas_sgemm(CblasRowMajor, TransA, TransB, + gsl::narrow_cast(M), + gsl::narrow_cast(N), + gsl::narrow_cast(K), + alpha, A, lda, B, ldb, + beta, C, gsl::narrow_cast(N)); } template <> diff --git a/onnxruntime/test/contrib_ops/gather_nd_op_test.cc b/onnxruntime/test/contrib_ops/gather_nd_op_test.cc new file mode 100644 index 0000000000000..f7af11bb1f4ec --- /dev/null +++ b/onnxruntime/test/contrib_ops/gather_nd_op_test.cc @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(GatherNDOpTest, GatherND_scaler_string_int32) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2}, {"h","k","o","z"}); + test1.AddInput("indices", {2}, {0,1}); + test1.AddOutput("output", {}, {"k"}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {6}, {"h","k","o","z","l","t"}); + test2.AddInput("indices", {1}, {3}); + test2.AddOutput("output", {}, {"z"}); + test2.Run(); + + OpTester test3("GatherND", 1, onnxruntime::kMSDomain); + test3.AddInput("data", {3,2}, {"h","k","o","z","l","t"}); + test3.AddInput("indices", {2}, {2,1}); + test3.AddOutput("output", {}, {"t"}); + test3.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_int64_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput ("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput ("indices", {2,2}, {0LL,0LL,1LL,1LL}); + test.AddOutput("output", {2}, {0LL,3LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_string_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {"a","b","c","d"}); + test.AddInput("indices", {2,2}, {0LL,0LL,1LL,1LL}); + test.AddOutput("output", {2}, {"a","d"}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_int64_int32) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput("indices", {2,2}, {0,0,1,1}); + test.AddOutput("output", {2}, {0LL,3LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_string_int32) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2,2}, {"egg","dance","air","bob","terry","smart","laugh","kite"}); + test1.AddInput("indices", {2,1,2}, {0,1,1,0}); + test1.AddOutput("output", {2,1,2}, {"air","bob","terry","smart"}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {3,3}, {"egg","dance","air","bob","terry","smart","laugh","kite","hop"}); + test2.AddInput("indices", {3,2}, {2,1,1,0,0,1}); + test2.AddOutput("output", {3}, {"kite","bob","dance"}); + test2.Run(); +} + +TEST(GatherNDOpTest, GatherND_slice_float_int64_t) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0.0f,0.1f,0.2f,0.3f}); + test.AddInput("indices", {2,1}, {1LL,0LL}); + test.AddOutput("output", {2,2}, {0.2f,0.3f,0.0f,0.1f}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_slice_double_int32_t) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0.0f,0.1f,0.2f,0.3f}); + test.AddInput("indices", {2,1}, {1LL,0LL}); + test.AddOutput("output", {2,2}, {0.2f,0.3f,0.0f,0.1f}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_3tensor_int64) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2,2}, {0LL,1LL,2LL,3LL,4LL,5LL,6LL,7LL}); + test1.AddInput("indices", {2,2}, {0LL,1LL,1LL,0LL}); + test1.AddOutput("output", {2,2}, {2LL,3LL,4LL,5LL}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test2.AddInput("indices", {2,3}, {0,0,1,1,0,1}); + test2.AddOutput("output", {2}, {1,5}); + test2.Run(); + + OpTester test3("GatherND", 1, onnxruntime::kMSDomain); + test3.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test3.AddInput("indices", {1,1}, {1LL}); + test3.AddOutput("output", {1,2,2}, {4,5,6,7}); + test3.Run(); +} + +TEST(GatherNDOpTest, GatherND_batched_index_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput("indices", {2,1,2}, {0LL,0LL,0LL,1LL}); + test.AddOutput("output", {2,1}, {0LL,1LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_batched_index_bool_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {true,false,false,true}); + test.AddInput("indices", {2,1,2}, {0LL,0LL,0LL,1LL}); + test.AddOutput("output", {2,1}, {true,false}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_sliced_index_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput("indices", {2,1,1}, {1LL,0LL}); + test.AddOutput("output", {2,1,2}, {2LL,3LL,0LL,1LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_sliced_index_string_int32) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {"ab","cde","f","ghi"}); + test.AddInput("indices", {2,1,1}, {1LL,0LL}); + test.AddOutput("output", {2,1,2}, {"f","ghi","ab","cde"}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_batched_3tensor_int64) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test1.AddInput("indices", {2,2,2}, {0LL,1LL,1LL,0LL,0LL,0LL,1LL,1LL}); + test1.AddOutput("output", {2,2,2}, {2,3,4,5,0,1,6,7}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test2.AddInput("indices", {2,2,3}, {0,0,1,1,0,1,0,1,1,1,1,0}); + test2.AddOutput("output", {2,2}, {1,5,3,6}); + test2.Run(); + + OpTester test3("GatherND", 1, onnxruntime::kMSDomain); + test3.AddInput("data", {2,2,2}, {0LL,1LL,2LL,3LL,4LL,5LL,6LL,7LL}); + test3.AddInput("indices", {2,1,1}, {1,0}); + test3.AddOutput("output", {2,1,2,2}, {4LL,5LL,6LL,7LL,0LL,1LL,2LL,3LL}); + test3.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index 8125bcb0ddc32..e2b741cffd44c 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -257,9 +257,9 @@ TEST(ExecutionFrameTest, MemPatternTest) { EXPECT_EQ(pattern.patterns.size(), pattern.locations.size()); EXPECT_EQ(pattern.patterns.size(), 1); auto p = pattern.GetPatterns(cpu_allocator->Info()); - EXPECT_EQ(p->PeakSize(), sizeof(float) * (4 + 6)); + EXPECT_EQ(p->PeakSize(), 2 * 64); // each allocation is 64-byte aligned EXPECT_EQ(p->GetBlock(3)->offset_, 0); - EXPECT_EQ(p->GetBlock(4)->offset_, sizeof(float) * 4); + EXPECT_EQ(p->GetBlock(4)->offset_, 64); } } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 7f7eb686ea39b..b323fa50c2f56 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -180,7 +180,7 @@ static Status SortTensorFileNames(std::vector> } Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, - const std::vector value_info, std::unordered_map& name_data_map, std::ostringstream& oss) { + const std::vector value_info, std::unordered_map& name_data_map, std::ostringstream& oss) { google::protobuf::io::FileInputStream f(test_data_pb_fd); f.SetCloseOnDelete(true); google::protobuf::io::CodedInputStream coded_input(&f); @@ -188,7 +188,7 @@ Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, Status st; int item_id = 1; for (proto::TraditionalMLData data; google::protobuf::util::ParseDelimitedFromCodedStream(&data, &coded_input, &clean_eof); ++item_id, data.Clear()) { - std::unique_ptr gvalue(nullptr, ReleaseONNXValue); + std::unique_ptr gvalue(nullptr, OrtReleaseValue); MLValue value; bool is_tensor = false; switch (data.values_case()) { @@ -223,9 +223,9 @@ Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, st = RichTypeProtoToMLValue(data.map_int64_to_double().v(), value); break; case proto::TraditionalMLData::kTensor: { - ONNXValue* temp_value; + OrtValue* temp_value; std::string s = data.tensor().SerializeAsString(); - ORT_THROW_ON_ERROR(OrtTensorProtoToONNXValue(env, s.data(), (int)s.size(), &temp_value)); + ORT_THROW_ON_ERROR(OrtTensorProtoToOrtValue(env, s.data(), (int)s.size(), &temp_value)); gvalue.reset(temp_value); is_tensor = true; } break; @@ -240,7 +240,7 @@ Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, if (value_name.empty()) value_name = value_info[name_data_map.size()].name(); - auto pv = name_data_map.insert(std::make_pair(value_name, is_tensor ? gvalue.release() : (ONNXValue*)new MLValue(value))); + auto pv = name_data_map.insert(std::make_pair(value_name, is_tensor ? gvalue.release() : (OrtValue*)new MLValue(value))); if (!pv.second) { st = Status(ONNXRUNTIME, FAIL, "duplicated test data name"); break; @@ -292,8 +292,8 @@ class OnnxTestCase : public ITestCase { } //If we cannot get input name from input_pbs, we'll use names like "data_0","data_1",... It's dirty hack // for https://github.com/onnx/onnx/issues/679 - ::onnxruntime::common::Status ConvertTestData(ONNXSession* session, const std::vector& test_data_pbs, - bool is_input, std::unordered_map& out); + ::onnxruntime::common::Status ConvertTestData(OrtSession* session, const std::vector& test_data_pbs, + bool is_input, std::unordered_map& out); std::string node_name_; std::once_flag model_parsed_; std::once_flag config_parsed_; @@ -333,7 +333,7 @@ class OnnxTestCase : public ITestCase { const std::string& GetTestCaseName() const override { return test_case_name_; } - ::onnxruntime::common::Status LoadTestData(ONNXSession* session, size_t id, std::unordered_map&, bool is_input) override; + ::onnxruntime::common::Status LoadTestData(OrtSession* session, size_t id, std::unordered_map&, bool is_input) override; }; Status OnnxTestCase::loadModelFile(const PATH_CHAR_TYPE* model_url, ONNX_NAMESPACE::ModelProto** model_pb) { @@ -463,7 +463,7 @@ static Status LoadTensors(const std::vector& pb_files, return Status::OK(); } -Status OnnxTestCase::LoadTestData(ONNXSession* session, size_t id, std::unordered_map& name_data_map, bool is_input) { +Status OnnxTestCase::LoadTestData(OrtSession* session, size_t id, std::unordered_map& name_data_map, bool is_input) { if (id >= test_data_dirs_.size()) return Status(ONNXRUNTIME, INVALID_ARGUMENT, "out of bound"); @@ -516,8 +516,8 @@ Status OnnxTestCase::LoadTestData(ONNXSession* session, size_t id, std::unordere return Status::OK(); } -Status OnnxTestCase::ConvertTestData(ONNXSession* session, const std::vector& test_data_pbs, - bool is_input, std::unordered_map& out) { +Status OnnxTestCase::ConvertTestData(OrtSession* session, const std::vector& test_data_pbs, + bool is_input, std::unordered_map& out) { bool has_valid_names = true; std::vector var_names(test_data_pbs.size()); for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { @@ -553,8 +553,8 @@ Status OnnxTestCase::ConvertTestData(ONNXSession* session, const std::vector& name_data_map, bool is_input) = 0; + virtual ::onnxruntime::common::Status LoadTestData(OrtSession* session, size_t id, std::unordered_map& name_data_map, bool is_input) = 0; virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; virtual const std::string& GetTestCaseName() const = 0; //a string to help identify the dataset diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 8f74bc5ea9a2d..93a64012db583 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -162,10 +162,10 @@ int real_main(int argc, char* argv[]) { std::unique_ptr env; { OrtEnv* t; - ONNXStatus* ost = OrtInitialize(logging_level, "Default", &t); + OrtStatus* ost = OrtInitialize(logging_level, "Default", &t); if (ost != nullptr) { fprintf(stderr, "Error creating environment: %s \n", OrtGetErrorMessage(ost)); - ReleaseONNXStatus(ost); + OrtReleaseStatus(ost); return -1; } env.reset(t); @@ -176,10 +176,10 @@ int real_main(int argc, char* argv[]) { std::unique_ptr default_allocator; { OrtAllocator* p; - ONNXStatus* ost = OrtCreateDefaultAllocator(&p); + OrtStatus* ost = OrtCreateDefaultAllocator(&p); if (ost != nullptr) { fprintf(stderr, "Error creating environment: %s \n", OrtGetErrorMessage(ost)); - ReleaseONNXStatus(ost); + OrtReleaseStatus(ost); return -1; } default_allocator.reset(p); @@ -205,7 +205,7 @@ int real_main(int argc, char* argv[]) { sf.AppendExecutionProvider(f); OrtReleaseObject(f); #else - fprintf(stderr, "CUDA is supported in this build"); + fprintf(stderr, "CUDA is not supported in this build"); return -1; #endif } @@ -216,7 +216,7 @@ int real_main(int argc, char* argv[]) { sf.AppendExecutionProvider(f); OrtReleaseObject(f); #else - fprintf(stderr, "Nuphar is supported in this build"); + fprintf(stderr, "Nuphar is not supported in this build"); return -1; #endif } @@ -227,7 +227,7 @@ int real_main(int argc, char* argv[]) { sf.AppendExecutionProvider(f); OrtReleaseObject(f); #else - fprintf(stderr, "MKL-DNN is supported in this build"); + fprintf(stderr, "MKL-DNN is not supported in this build"); return -1; #endif } @@ -309,13 +309,9 @@ int real_main(int argc, char* argv[]) { {"operator_rnn_single_layer", "disable reason"}, {"prelu_broadcast", "disable reason"}, {"prelu_example", "disable reason"}, - {"maxunpool_export_with_output_shape", "opset 9 not supported yet"}, - {"maxunpool_export_without_output_shape", "opset 9 not supported yet"}, {"upsample_nearest", "opset 9 not supported yet"}, {"onehot_with_axis", "opset 9 not supported yet"}, {"onehot_without_axis", "opset 9 not supported yet"}, // also has bug in current test re: output type. Spandan to fix. - {"sinh", "opset 9 not supported yet"}, - {"cosh", "opset 9 not supported yet"}, {"asinh", "opset 9 not supported yet"}, {"acosh", "opset 9 not supported yet"}, {"atanh", "opset 9 not supported yet"}, diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc index 7fac9800f13c7..ad3bbfd032bbf 100644 --- a/onnxruntime/test/onnx/runner.cc +++ b/onnxruntime/test/onnx/runner.cc @@ -88,7 +88,7 @@ void PTestRunner::OnTaskFinished(size_t, EXECUTE_RESULT, ORT_CALLBACK_INSTANCE p } } -PTestRunner::PTestRunner(ONNXSession* session1, +PTestRunner::PTestRunner(OrtSession* session1, ITestCase* c, PThreadPool tpool, TestCaseCallBack on_finished1) : DataRunner(session1, c->GetTestCaseName(), c, on_finished1), next_test_to_run(0), finished(0), tpool_(tpool) { } @@ -282,12 +282,12 @@ std::vector LoadTests(const std::vectorGetTestCaseName(), c, on_finished1), repeat_count_(repeat_count) { } -DataRunner::DataRunner(ONNXSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1) : test_case_name_(test_case_name1), c_(c), session(session1), on_finished(on_finished1), default_allocator(MockedOrtAllocator::Create()) { +DataRunner::DataRunner(OrtSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1) : test_case_name_(test_case_name1), c_(c), session(session1), on_finished(on_finished1), default_allocator(MockedOrtAllocator::Create()) { std::string s; c->GetNodeName(&s); result = std::make_shared(c->GetDataCount(), EXECUTE_RESULT::UNKNOWN_ERROR, s); @@ -308,12 +308,12 @@ void DataRunner::RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_r OnTaskFinished(task_id, res, pci); } -std::pair CompareGenericValue(const ONNXValue* o, const ONNXValue* expected_mlvalue, double per_sample_tolerance, double relative_per_sample_tolerance, +std::pair CompareGenericValue(const OrtValue* o, const OrtValue* expected_mlvalue, double per_sample_tolerance, double relative_per_sample_tolerance, bool post_processing) { return onnxruntime::CompareMLValue(*(MLValue*)o, *(MLValue*)expected_mlvalue, per_sample_tolerance, relative_per_sample_tolerance, post_processing); } EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { - std::unordered_map feeds; + std::unordered_map feeds; common::Status status = c_->LoadTestData(session, task_id, feeds, true); if (!status.IsOK()) { LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); @@ -335,14 +335,14 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { TIME_SPEC start_time, end_time; GetMonotonicTimeCounter(&start_time); std::vector input_names(feeds.size()); - std::vector input_values(feeds.size()); + std::vector input_values(feeds.size()); size_t input_index = 0; for (auto& kvp : feeds) { input_names[input_index] = kvp.first.c_str(); input_values[input_index] = kvp.second; ++input_index; } - std::vector output_values(output_count); + std::vector output_values(output_count); { std::vector output_names_raw_ptr(output_count); for (size_t i = 0; i != output_count; ++i) { @@ -351,9 +351,9 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { auto onnx_status = OrtRunInference(session, nullptr, input_names.data(), input_values.data(), input_index, output_names_raw_ptr.data(), output_count, output_values.data()); if (onnx_status != nullptr) { std::string onnx_runtime_error_message = OrtGetErrorMessage(onnx_status); - ReleaseONNXStatus(onnx_status); + OrtReleaseStatus(onnx_status); for (auto& kvp : feeds) { - ReleaseONNXValue(kvp.second); + OrtReleaseValue(kvp.second); } throw std::runtime_error(onnx_runtime_error_message); } @@ -361,7 +361,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { GetMonotonicTimeCounter(&end_time); AccumulateTimeSpec(&spent_time_, &start_time, &end_time); for (auto& kvp : feeds) { - ReleaseONNXValue(kvp.second); + OrtReleaseValue(kvp.second); } if (!status.IsOK()) { LOGF_DEFAULT(ERROR, "%s:%s\n", test_case_name_.c_str(), status.ErrorMessage().c_str()); @@ -385,13 +385,13 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { } //TODO: if there are no output value files, just skip the validation - std::unordered_map expected_output_values; + std::unordered_map expected_output_values; status = c_->LoadTestData(session, task_id, expected_output_values, false); if (!status.IsOK()) { LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); return StatusCodeToExecuteResult(status.Code()); } - std::unordered_map name_fetch_output_map; + std::unordered_map name_fetch_output_map; std::unordered_map name_output_value_info_proto; int i = 0; for (auto& output_name : output_names) { @@ -404,7 +404,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { EXECUTE_RESULT res = EXECUTE_RESULT::SUCCESS; for (auto& output : expected_output_values) { - ONNXValue* expected_output_value = output.second; + OrtValue* expected_output_value = output.second; const std::string& output_name = output.first; auto iter = name_fetch_output_map.find(output_name); if (iter == name_fetch_output_map.end()) { @@ -412,7 +412,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { LOGF_DEFAULT(ERROR, "cannot find %s in the outputs", output_name.c_str()); break; } - ONNXValue* actual_output_value = iter->second; + OrtValue* actual_output_value = iter->second; std::pair ret = CompareGenericValue(actual_output_value, expected_output_value, per_sample_tolerance, relative_per_sample_tolerance, post_procesing); COMPARE_RESULT compare_result = ret.first; if (compare_result == COMPARE_RESULT::SUCCESS) { @@ -461,10 +461,10 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { } } for (auto& kvp : expected_output_values) { - ReleaseONNXValue(kvp.second); + OrtReleaseValue(kvp.second); } - for (ONNXValue* p : output_values) { - ReleaseONNXValue(p); + for (OrtValue* p : output_values) { + OrtReleaseValue(p); } return res; } @@ -492,8 +492,8 @@ void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper } auto sf2 = sf.clone(); sf2.SetSessionLogId(info->GetTestCaseName().c_str()); - std::unique_ptr session_object( - sf2.OrtCreateInferenceSession(info->GetModelUrl()), ReleaseONNXSession); + std::unique_ptr session_object( + sf2.OrtCreateInferenceSession(info->GetModelUrl()), OrtReleaseSession); LOGF_DEFAULT(INFO, "testing %s\n", info->GetTestCaseName().c_str()); //temp hack. Because we have no resource control. We may not have enough memory to run this test in parallel if (info->GetTestCaseName() == "coreml_FNS-Candy_ImageNet") diff --git a/onnxruntime/test/onnx/runner.h b/onnxruntime/test/onnx/runner.h index 9691e52e3844b..f3dfd852cdf42 100644 --- a/onnxruntime/test/onnx/runner.h +++ b/onnxruntime/test/onnx/runner.h @@ -41,18 +41,18 @@ class DataRunner { ::onnxruntime::TIME_SPEC spent_time_; private: - ONNXSession* session; + OrtSession* session; CALL_BACK on_finished; OrtAllocatorInterface** const default_allocator; EXECUTE_RESULT RunTaskImpl(size_t task_id); ORT_DISALLOW_COPY_AND_ASSIGNMENT(DataRunner); public: - DataRunner(ONNXSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1); + DataRunner(OrtSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1); virtual void OnTaskFinished(size_t task_id, EXECUTE_RESULT res, ORT_CALLBACK_INSTANCE pci) noexcept = 0; void RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_result); virtual ~DataRunner() { - ReleaseONNXSession(session); + OrtReleaseSession(session); OrtReleaseObject(default_allocator); } @@ -98,7 +98,7 @@ class SeqTestRunner : public DataRunner { size_t repeat_count_; public: - SeqTestRunner(ONNXSession* session1, + SeqTestRunner(OrtSession* session1, ITestCase* c, size_t repeat_count, TestCaseCallBack on_finished1); @@ -115,7 +115,7 @@ class PTestRunner : public DataRunner { public: void Start(ORT_CALLBACK_INSTANCE pci, size_t concurrent_runs) override; - PTestRunner(ONNXSession* session1, + PTestRunner(OrtSession* session1, ITestCase* c, PThreadPool tpool, TestCaseCallBack on_finished1); diff --git a/onnxruntime/test/onnxruntime_exec/Runtime.h b/onnxruntime/test/onnxruntime_exec/Runtime.h index bbf1aed93479f..b74ee583f19e7 100644 --- a/onnxruntime/test/onnxruntime_exec/Runtime.h +++ b/onnxruntime/test/onnxruntime_exec/Runtime.h @@ -273,7 +273,7 @@ class WinMLRuntime { ctensor = &output.Get(); ONNX_NAMESPACE::ValueInfoProto expected_output_info = (*outputMeta)[i]->ToProto(); - std::pair ret = VerifyValueInfo(expected_output_info, (ONNXValue*)&output); + std::pair ret = VerifyValueInfo(expected_output_info, (OrtValue*)&output); COMPARE_RESULT compare_result = ret.first; compare_result = ret.first; if (compare_result != COMPARE_RESULT::SUCCESS) { diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index e8845542390fe..fa1b369b2d2d6 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -163,11 +163,11 @@ TEST(MathOpTest, Sub_int32) { } TEST(MathOpTest, Sub_int64) { - OpTester test("Sub"); - test.AddInput("A", { 3 }, { 1, 5, 6 }); - test.AddInput("B", { 3 }, { 4, 5, 3 }); - test.AddOutput("C", { 3 }, { -3, 0, 3 }); - test.Run(); + OpTester test("Sub"); + test.AddInput("A", {3}, {1, 5, 6}); + test.AddInput("B", {3}, {4, 5, 3}); + test.AddOutput("C", {3}, {-3, 0, 3}); + test.Run(); } TEST(MathOpTest, Sub) { @@ -212,11 +212,11 @@ TEST(MathOpTest, Mul_int32) { } TEST(MathOpTest, Mul_int64) { - OpTester test("Mul"); - test.AddInput("A", { 3 }, { 3, 6, -3 }); - test.AddInput("B", { 3 }, { 4, -3, -2 }); - test.AddOutput("C", { 3 }, { 12, -18, 6 }); - test.Run(); + OpTester test("Mul"); + test.AddInput("A", {3}, {3, 6, -3}); + test.AddInput("B", {3}, {4, -3, -2}); + test.AddOutput("C", {3}, {12, -18, 6}); + test.Run(); } TEST(MathOpTest, Mul) { @@ -246,11 +246,11 @@ TEST(MathOpTest, Div_int32) { } TEST(MathOpTest, Div_int64) { - OpTester test("Div"); - test.AddInput("A", { 3 }, { 4, 8, 8 }); - test.AddInput("B", { 3 }, { 2, 3, 4 }); - test.AddOutput("C", { 3 }, { 2, 2, 2 }); - test.Run(); + OpTester test("Div"); + test.AddInput("A", {3}, {4, 8, 8}); + test.AddInput("B", {3}, {2, 3, 4}); + test.AddOutput("C", {3}, {2, 2, 2}); + test.Run(); } TEST(MathOpTest, Div) { @@ -819,6 +819,16 @@ TEST(MathOpTest, Atan) { TrigTest(test, {-10.0f, -5.0f, 0.0f, 5.0f, 10.0f}); } +TEST(MathOpTest, Sinh) { + OpTester test("Sinh", 9); + TrigTest(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); +} + +TEST(MathOpTest, Cosh) { + OpTester test("Cosh", 9); + TrigTest(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); +} + TEST(MathOpTest, Expand_8_3x3) { OpTester test("Expand", 8); test.AddInput("data_0", {1}, {1.0f}); diff --git a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc index 164e143fbcf3d..8d8b5b04e0a16 100644 --- a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc +++ b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include + #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" #include "gsl/gsl" @@ -8,19 +10,23 @@ using namespace std; namespace onnxruntime { namespace test { -TEST(MLOpTest, ArrayFeatureExtractorTest) { - OpTester test("ArrayFeatureExtractor", 1, onnxruntime::kMLDomain); +class ArrayFeatureExtractorTest : public ::testing::Test { + protected: + OpTester test_{"ArrayFeatureExtractor", 1, onnxruntime::kMLDomain}; +}; + +TEST_F(ArrayFeatureExtractorTest, Basic) { const int N = 3; const std::vector X = {0.8f, -1.5f, 2.0f, 3.8f, -4.0f, 5.0f, 6.8f, -7.5f, 8.0f, 9.8f, -9.0f, 4.0f, 4.8f, -4.5f, 4.0f, 4.8f, -4.0f, 4.0f}; const int kCols = 6; const vector x_dims = {N, kCols}; - test.AddInput("X", x_dims, X); + test_.AddInput("X", x_dims, X); const std::vector Y = {1L, 2L, 4L}; const vector y_dims = {1, 3}; - test.AddInput("Y", y_dims, Y); + test_.AddInput("Y", y_dims, Y); // prepare expected output vector expected_output; @@ -31,9 +37,76 @@ TEST(MLOpTest, ArrayFeatureExtractorTest) { } } const vector expected_dims{N, gsl::narrow_cast(Y.size())}; - test.AddOutput("Z", expected_dims, expected_output); + test_.AddOutput("Z", expected_dims, expected_output); + + test_.Run(); +} + +TEST_F(ArrayFeatureExtractorTest, HigherDimensionalX) { + const std::vector x_dims{2, 3, 4, 5}; + const int64_t x_size = std::accumulate( + x_dims.begin(), x_dims.end(), static_cast(1), std::multiplies<>{}); + const std::vector X = [x_size]() { + std::vector v(x_size); + std::iota(v.begin(), v.end(), 0); + return v; + }(); + test_.AddInput("X", x_dims, X); + + const std::vector Y{0, 1, 1, 2}; + const int64_t y_size = gsl::narrow_cast(Y.size()); + const std::vector y_dims{1, y_size}; + test_.AddInput("Y", y_dims, Y); + + // prepare expected output + const std::vector z_dims = [&x_dims, y_size]() { + std::vector v{x_dims}; + v[v.size() - 1] = y_size; + return v; + }(); + const std::vector Z = [&x_dims, x_size, y_size, &X, &Y]() { + const int64_t x_last_dim_size = x_dims.back(); // stride + const int64_t x_leading_dims_size = x_size / x_last_dim_size; + std::vector v(x_leading_dims_size * y_size); + int32_t* v_output = v.data(); + for (int64_t x_idx = 0; x_idx < x_size; x_idx += x_last_dim_size) { + for (int64_t y_idx = 0; y_idx < y_size; ++y_idx) { + *(v_output++) = X[x_idx + Y[y_idx]]; + } + } + return v; + }(); + test_.AddOutput("Z", z_dims, Z); + + test_.Run(); +} + +TEST_F(ArrayFeatureExtractorTest, OneDimensionalX) { + test_.AddInput("X", {1}, {42}); + test_.AddInput("Y", {1, 3}, {0, 0, 0}); + test_.AddOutput("Z", {1, 3}, {42, 42, 42}); + test_.Run(); +} + +TEST_F(ArrayFeatureExtractorTest, InvalidInputEmptyX) { + test_.AddInput("X", {0}, {}); + test_.AddInput("Y", {1}, {1}); + test_.AddOutput("Z", {0}, {}); + test_.Run(OpTester::ExpectResult::kExpectFailure); +} + +TEST_F(ArrayFeatureExtractorTest, InvalidInputEmptyY) { + test_.AddInput("X", {1}, {1}); + test_.AddInput("Y", {0}, {}); + test_.AddOutput("Z", {0}, {}); + test_.Run(OpTester::ExpectResult::kExpectFailure); +} - test.Run(); +TEST_F(ArrayFeatureExtractorTest, InvalidInputOutOfBoundsY) { + test_.AddInput("X", {2, 2}, {1, 2, 3, 4}); + test_.AddInput("Y", {1}, {10}); + test_.AddOutput("Z", {0}, {}); + test_.Run(OpTester::ExpectResult::kExpectFailure, "index is out of range"); } } // namespace test diff --git a/onnxruntime/test/providers/cpu/nn/unpool_op_test.cc b/onnxruntime/test/providers/cpu/nn/unpool_op_test.cc new file mode 100644 index 0000000000000..fd9b20ba9c3e9 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/unpool_op_test.cc @@ -0,0 +1,382 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(UnpoolTest, MaxUnPool1D) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 8}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0}; + + std::vector inputDims = {3}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", std::vector{2, 2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + std::vector inputDims = {4}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + + std::vector t_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector t_dims = {1, 1, 2, 2, 2}; + + std::vector i_vals = {1, 3, 24, 30, 32, 38, 60, 62}; + std::vector i_dims = {1, 1, 2, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4, 4}; + std::vector expectedDims_Size = {5}; + + std::vector expected_vals = + { + //slice 1 + 0, 1, 0, 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 2 + 0, 0, 0, 0, + 0, 0, 0, 0, + 3, 0, 0, 0, + 0, 0, 4, 0, + + //slice 3 + 5, 0, 0, 0, + 0, 0, 6, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 4 + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 7, 0, 8, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", expectedDims_Size, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool1D_Without_OutputShape) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 8}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D_Without_OutputShape) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D_Without_OutputShape) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + + std::vector t_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector t_dims = {1, 1, 2, 2, 2}; + + std::vector i_vals = {1, 3, 24, 30, 32, 38, 60, 62}; + std::vector i_dims = {1, 1, 2, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4, 4}; + + std::vector expected_vals = + { + //slice 1 + 0, 1, 0, 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 2 + 0, 0, 0, 0, + 0, 0, 0, 0, + 3, 0, 0, 0, + 0, 0, 4, 0, + + //slice 3 + 5, 0, 0, 0, + 0, 0, 6, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 4 + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 7, 0, 8, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool1D_Padding) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + test.AddAttribute("pads", vector{1, 0}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 7}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("YP", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D_Padding) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2}); + test.AddAttribute("pads", vector{1, 1, 0, 0}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 3, 3}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D_Padding) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + test.AddAttribute("pads", vector{0, 1, 1, 0, 0, 0}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 1, 2, 2}; + + std::vector i_vals = {1, 4, 8, 12}; + std::vector i_dims = {1, 1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 2, 3, 3}; + + std::vector expected_vals = { + 0, 1, 0, + 0, 2, 0, + 0, 0, 3, + 0, 0, 0, + 4, 0, 0, + 0, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool1D_WithPaddedOutput) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 10}; + std::vector expected_vals = {0, 0, 1, 0, 2, 3, 0, 4, 0, 0}; + + std::vector inputDims = {3}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D_WithPaddedOutput) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", std::vector{2, 2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 8, 10}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 5, 5}; + std::vector expected_vals = { + 0, 1, 0, 2, 0, + 0, 0, 0, 0, 0, + 3, 0, 4, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0}; + + std::vector inputDims = {4}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D_WithPaddedOutput) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + + std::vector t_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector t_dims = {1, 1, 2, 2, 2}; + + std::vector i_vals = {1, 3, 24, 30, 32, 38, 60, 62}; + std::vector i_dims = {1, 1, 2, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4, 5}; + std::vector expectedDims_Size = {5}; + + std::vector expected_vals = + { + //slice 1 + 0, 1, 0, 2, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + + // slice 2 + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, + 0, 0, 4, 0, 0, + + //slice 3 + 5, 0, 0, 0, 0, + 0, 0, 6, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + + // slice 4 + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 7, 0, 8, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", expectedDims_Size, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 80f7f6c81f34d..ea6ea135bcd20 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -24,7 +24,6 @@ '|test_atanh_example_cpu.*' '|test_convtranspose_1d_cpu.*' '|test_convtranspose_3d_cpu.*' -'|test_cosh_cpu.*' '|test_cosh_example_cpu.*' '|test_dynamic_slice_cpu.*' '|test_dynamic_slice_default_axes_cpu.*' @@ -35,15 +34,12 @@ '|test_eyelike_with_dtype_cpu.*' '|test_eyelike_without_dtype_cpu.*' '|test_gru_seq_length_cpu.*' -'|test_maxunpool_export_with_output_shape_cpu.*' -'|test_maxunpool_export_without_output_shape_cpu.*' '|test_onehot_with_axis_cpu.*' '|test_onehot_without_axis_cpu.*' '|test_scan_sum_cpu.*' '|test_scatter_with_axis_cpu.*' '|test_scatter_without_axis_cpu.*' '|test_sign_cpu.*' -'|test_sinh_cpu.*' '|test_sinh_example_cpu.*' '|test_AvgPool1d_cpu.*' '|test_AvgPool1d_stride_cpu.*' diff --git a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c index b26bb4df91857..7aba00362f0ef 100644 --- a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c +++ b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c @@ -8,11 +8,11 @@ #define ORT_ABORT_ON_ERROR(expr) \ do { \ - ONNXStatus* onnx_status = (expr); \ + OrtStatus* onnx_status = (expr); \ if (onnx_status != NULL) { \ const char* msg = OrtGetErrorMessage(onnx_status); \ fprintf(stderr, "%s\n", msg); \ - ReleaseONNXStatus(onnx_status); \ + OrtReleaseStatus(onnx_status); \ abort(); \ } \ } while (0); @@ -94,7 +94,7 @@ static int read_png_file(const char* input_file, size_t* height, size_t* width, /** * \param tensor should be a float tensor in [N,C,H,W] format */ -static int write_tensor_to_png_file(ONNXValue* tensor, const char* output_file) { +static int write_tensor_to_png_file(OrtValue* tensor, const char* output_file) { struct OrtTensorTypeAndShapeInfo* shape_info; ORT_ABORT_ON_ERROR(OrtGetTensorShapeAndType(tensor, &shape_info)); size_t dim_count = OrtGetNumOfDimensions(shape_info); @@ -132,7 +132,7 @@ static void usage() { printf("usage: \n"); } -int run_inference(ONNXSession* session, const char* input_file, const char* output_file) { +int run_inference(OrtSession* session, const char* input_file, const char* output_file) { size_t input_height; size_t input_width; float* model_input; @@ -151,28 +151,28 @@ int run_inference(ONNXSession* session, const char* input_file, const char* outp const size_t input_shape_len = sizeof(input_shape) / sizeof(input_shape[0]); const size_t model_input_len = model_input_ele_count * sizeof(float); - ONNXValue* input_tensor = NULL; - ORT_ABORT_ON_ERROR(OrtCreateTensorWithDataAsONNXValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); + OrtValue* input_tensor = NULL; + ORT_ABORT_ON_ERROR(OrtCreateTensorWithDataAsOrtValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); assert(input_tensor != NULL); assert(OrtIsTensor(input_tensor) != 0); - ReleaseOrtAllocatorInfo(allocator_info); + OrtReleaseAllocatorInfo(allocator_info); const char* input_names[] = {"inputImage"}; const char* output_names[] = {"outputImage"}; - ONNXValue* output_tensor = NULL; - ORT_ABORT_ON_ERROR(OrtRunInference(session, NULL, input_names, (const ONNXValue* const*)&input_tensor, 1, output_names, 1, &output_tensor)); + OrtValue* output_tensor = NULL; + ORT_ABORT_ON_ERROR(OrtRunInference(session, NULL, input_names, (const OrtValue* const*)&input_tensor, 1, output_names, 1, &output_tensor)); assert(output_tensor != NULL); assert(OrtIsTensor(output_tensor) != 0); int ret = 0; if (write_tensor_to_png_file(output_tensor, output_file) != 0) { ret = -1; } - ReleaseONNXValue(output_tensor); - ReleaseONNXValue(input_tensor); + OrtReleaseValue(output_tensor); + OrtReleaseValue(input_tensor); free(model_input); return ret; } -void verify_input_output_count(ONNXSession* session) { +void verify_input_output_count(OrtSession* session) { size_t count; ORT_ABORT_ON_ERROR(OrtInferenceSessionGetInputCount(session, &count)); assert(count == 1); @@ -203,12 +203,12 @@ int main(int argc, char* argv[]) { #ifdef USE_CUDA enable_cuda(session_option); #endif - ONNXSession* session; + OrtSession* session; ORT_ABORT_ON_ERROR(OrtCreateInferenceSession(env, model_path, session_option, &session)); verify_input_output_count(session); int ret = run_inference(session, input_file, output_file); OrtReleaseObject(session_option); - ReleaseONNXSession(session); + OrtReleaseSession(session); OrtReleaseObject(env); if (ret != 0) { fprintf(stderr, "fail\n"); diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc index ef4921d5996a9..ab97260e95c4a 100644 --- a/onnxruntime/test/shared_lib/test_allocator.cc +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -13,8 +13,8 @@ TEST_F(CApiTest, allocation_info) { ORT_THROW_ON_ERROR(OrtCreateAllocatorInfo("Cpu", OrtArenaAllocator, 0, OrtMemTypeDefault, &info1)); ORT_THROW_ON_ERROR(OrtCreateCpuAllocatorInfo(OrtArenaAllocator, OrtMemTypeDefault, &info2)); ASSERT_EQ(0, OrtCompareAllocatorInfo(info1, info2)); - ReleaseOrtAllocatorInfo(info1); - ReleaseOrtAllocatorInfo(info2); + OrtReleaseAllocatorInfo(info1); + OrtReleaseAllocatorInfo(info2); } TEST_F(CApiTest, DefaultAllocator) { diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index b5d3c10fb9791..e1b2156bf9e9e 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -13,20 +13,20 @@ using namespace onnxruntime; -void RunSession(OrtAllocator* env, ONNXSession* session_object, +void RunSession(OrtAllocator* env, OrtSession* session_object, const std::vector& dims_x, const std::vector& values_x, const std::vector& dims_y, const std::vector& values_y) { - std::unique_ptr value_x(nullptr, ReleaseONNXValue); - std::vector inputs(1); - inputs[0] = OrtCreateTensorAsONNXValue(env, dims_x, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + std::unique_ptr value_x(nullptr, OrtReleaseValue); + std::vector inputs(1); + inputs[0] = OrtCreateTensorAsOrtValue(env, dims_x, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); value_x.reset(inputs[0]); void* raw_data; ORT_THROW_ON_ERROR(OrtGetTensorMutableData(inputs[0], &raw_data)); memcpy(raw_data, values_x.data(), values_x.size() * sizeof(values_x[0])); std::vector input_names{"X"}; - ONNXValue* output_tensor = nullptr; + OrtValue* output_tensor = nullptr; const char* output_names[] = {"Y"}; ORT_THROW_ON_ERROR(OrtRunInference(session_object, NULL, input_names.data(), inputs.data(), inputs.size(), output_names, 1, &output_tensor)); ASSERT_NE(output_tensor, nullptr); @@ -50,7 +50,7 @@ void RunSession(OrtAllocator* env, ONNXSession* session_object, for (size_t i = 0; i != total_len; ++i) { ASSERT_EQ(values_y[i], f[i]); } - ReleaseONNXValue(output_tensor); + OrtReleaseValue(output_tensor); } template @@ -98,7 +98,7 @@ void TestInference(OrtEnv* env, T model_uri, if (custom_op) { sf.AddCustomOp("libonnxruntime_custom_op_shared_lib_test.so"); } - std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), ReleaseONNXSession); + std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), OrtReleaseSession); std::unique_ptr default_allocator(MockedOrtAllocator::Create()); // Now run RunSession(default_allocator.get(), inference_session.get(), dims_x, values_x, expected_dims_y, expected_values_y); @@ -147,10 +147,10 @@ TEST_F(CApiTest, DISABLED_custom_op) { #ifdef ORT_RUN_EXTERNAL_ONNX_TESTS TEST_F(CApiTest, create_session_without_session_option) { constexpr PATH_TYPE model_uri = TSTR("../models/opset8/test_squeezenet/model.onnx"); - ONNXSession* ret; + OrtSession* ret; ORT_THROW_ON_ERROR(::OrtCreateInferenceSession(env, model_uri, nullptr, &ret)); ASSERT_NE(nullptr, ret); - ReleaseONNXSession(ret); + OrtReleaseSession(ret); } #endif TEST_F(CApiTest, create_tensor) { @@ -158,8 +158,8 @@ TEST_F(CApiTest, create_tensor) { size_t expected_len = 2; std::unique_ptr default_allocator(MockedOrtAllocator::Create()); { - std::unique_ptr tensor( - OrtCreateTensorAsONNXValue(default_allocator.get(), {expected_len}, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING), ReleaseONNXValue); + std::unique_ptr tensor( + OrtCreateTensorAsOrtValue(default_allocator.get(), {expected_len}, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING), OrtReleaseValue); ORT_THROW_ON_ERROR(OrtFillStringTensor(tensor.get(), s, expected_len)); std::unique_ptr shape_info; { @@ -185,9 +185,9 @@ TEST_F(CApiTest, create_tensor_with_data) { OrtAllocatorInfo* info; ORT_THROW_ON_ERROR(OrtCreateAllocatorInfo("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault, &info)); std::vector dims = {4}; - std::unique_ptr tensor( - OrtCreateTensorWithDataAsONNXValue(info, values, values_length * sizeof(float), dims, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT), ReleaseONNXValue); - ReleaseOrtAllocatorInfo(info); + std::unique_ptr tensor( + OrtCreateTensorWithDataAsOrtValue(info, values, values_length * sizeof(float), dims, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT), OrtReleaseValue); + OrtReleaseAllocatorInfo(info); void* new_pointer; ORT_THROW_ON_ERROR(OrtGetTensorMutableData(tensor.get(), &new_pointer)); ASSERT_EQ(new_pointer, values); diff --git a/onnxruntime/test/shared_lib/test_io_types.cc b/onnxruntime/test/shared_lib/test_io_types.cc index 60caf4f18c4c1..ceebc782c66aa 100644 --- a/onnxruntime/test/shared_lib/test_io_types.cc +++ b/onnxruntime/test/shared_lib/test_io_types.cc @@ -6,7 +6,7 @@ using namespace onnxruntime; -static void TestModelInfo(const ONNXSession* inference_session, bool is_input, const std::vector& dims) { +static void TestModelInfo(const OrtSession* inference_session, bool is_input, const std::vector& dims) { size_t input_count; if (is_input) { ORT_THROW_ON_ERROR(OrtInferenceSessionGetInputCount(inference_session, &input_count)); @@ -28,7 +28,7 @@ static void TestModelInfo(const ONNXSession* inference_session, bool is_input, c const OrtTensorTypeAndShapeInfo* p = OrtCastTypeInfoToTensorInfo(input_type_info.get()); ASSERT_NE(nullptr, p); - enum OrtTensorElementDataType ele_type = OrtGetTensorElementType(p); + enum ONNXTensorElementDataType ele_type = OrtGetTensorElementType(p); ASSERT_EQ(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, ele_type); ASSERT_EQ(dims.size(), OrtGetNumOfDimensions(p)); std::vector real_dims(dims.size()); @@ -39,7 +39,7 @@ static void TestModelInfo(const ONNXSession* inference_session, bool is_input, c TEST_F(CApiTest, input_output_type_info) { SessionOptionsWrapper sf(env); constexpr PATH_TYPE model_uri = TSTR("../models/opset8/test_squeezenet/model.onnx"); - std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), ReleaseONNXSession); + std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), OrtReleaseSession); TestModelInfo(inference_session.get(), true, {1, 3, 224, 224}); TestModelInfo(inference_session.get(), false, {1, 1000, 1, 1}); } diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc index 503eef554a155..24459e5e80803 100644 --- a/onnxruntime/test/util/compare_mlvalue.cc +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -26,7 +26,7 @@ using namespace onnxruntime; namespace { -OrtTensorElementDataType CApiElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { +ONNXTensorElementDataType CApiElementTypeFromProto(int type) { switch (type) { CASE_TYPE(FLOAT) CASE_TYPE(UINT8) @@ -64,14 +64,20 @@ std::pair CompareFloatResult(const Tensor& outvalue const double real_value = post_processing ? std::max(0.0, std::min(255.0, real_output[di])) : real_output[di]; const double diff = fabs(expected_output[di] - real_value); - const double rtol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected_output[di]); - if (diff > rtol || (std::isnan(diff) && !std::isnan(expected_output[di]))) { + const double tol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected_output[di]); + if (diff > tol || (std::isnan(diff) && !std::isnan(expected_output[di]))) { res.first = COMPARE_RESULT::RESULT_DIFFERS; // update error message if this is a larger diff if (diff > max_diff || (std::isnan(diff) && !std::isnan(max_diff))) { + int64_t expected_int = 0; + int64_t real_int = 0; + memcpy(&expected_int, &expected_output[di], sizeof(FLOAT_TYPE)); + memcpy(&real_int, &real_output[di], sizeof(FLOAT_TYPE)); + std::ostringstream oss; - oss << "expected " << expected_output[di] << ", got " << real_value - << ", diff: " << diff << ", tol=" << rtol << "."; + oss << std::hex << "expected " << expected_output[di] << " (" << expected_int << "), got " + << real_value << " (" << real_int << ")" + << ", diff: " << diff << ", tol=" << tol << "."; res.second = oss.str(); max_diff = diff; } @@ -321,7 +327,7 @@ std::pair CompareMLValue(const MLValue& o, const ML per_sample_tolerance, relative_per_sample_tolerance, post_processing); } -std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const ONNXValue* o) { +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const OrtValue* o) { if (!v.has_type()) return std::make_pair(COMPARE_RESULT::SUCCESS, ""); if (v.type().has_tensor_type()) { if (OrtIsTensor(o) == 0) { @@ -339,8 +345,8 @@ std::pair VerifyValueInfo(const ONNX_NAMESPACE::Val ORT_THROW_ON_ERROR(OrtGetTensorShapeAndType(o, &t1)); info.reset(t1); } - OrtTensorElementDataType real_type = OrtGetTensorElementType(info.get()); - OrtTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type()); + ONNXTensorElementDataType real_type = OrtGetTensorElementType(info.get()); + ONNXTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type()); if (real_type != expected_type) { return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); } diff --git a/onnxruntime/test/util/include/test/compare_mlvalue.h b/onnxruntime/test/util/include/test/compare_mlvalue.h index 6dbabba1355d1..32024287a76d4 100644 --- a/onnxruntime/test/util/include/test/compare_mlvalue.h +++ b/onnxruntime/test/util/include/test/compare_mlvalue.h @@ -23,5 +23,5 @@ std::pair CompareMLValue(const MLValue& real, const double relative_per_sample_tolerance, bool post_processing); //verify if the 'value' matches the 'expected' ValueInfoProto. 'value' is a model output -std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const ONNXValue* value); +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const OrtValue* value); } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/test_allocator.h b/onnxruntime/test/util/include/test_allocator.h index 1ef5929e1d8b2..660320c901647 100644 --- a/onnxruntime/test/util/include/test_allocator.h +++ b/onnxruntime/test/util/include/test_allocator.h @@ -47,7 +47,7 @@ MockedOrtAllocator() : ref_count_(1), memory_inuse(0) { } ~MockedOrtAllocator() { assert(ref_count_ == 0); - ReleaseOrtAllocatorInfo(cpuAllocatorInfo); + OrtReleaseAllocatorInfo(cpuAllocatorInfo); } public: diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 48ee7a01604f4..870c5295e09d3 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -24,6 +24,11 @@ test_data_url = 'https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip' test_data_checksum = 'a966def7447f4ff04f5665bca235b3f3' +class BuildError(Exception): + """Error from running build steps.""" + def __init__(self, *messages): + super().__init__("\n".join(messages)) + def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.", usage=''' @@ -84,7 +89,7 @@ def parse_arguments(): parser.add_argument("--ctest_path", default="ctest", help="Path to the CTest program.") parser.add_argument("--skip_submodule_sync", action='store_true', help="Don't do a 'git submodule update'. Makes the Update phase faster.") - parser.add_argument("--use_jemalloc", action='store_true', help="use jemalloc.") + parser.add_argument("--use_jemalloc", action='store_true', help="Use jemalloc.") parser.add_argument("--use_openblas", action='store_true', help="Build with OpenBLAS.") parser.add_argument("--use_mkldnn", action='store_true', help="Build with MKLDNN.") parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.") @@ -95,14 +100,21 @@ def parse_arguments(): parser.add_argument("--use_llvm", action="store_true", help="Build tvm with llvm") parser.add_argument("--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.") parser.add_argument("--llvm_path", help="Path to llvm dir") - parser.add_argument("--azure_sas_key", help="azure storage sas key, starts with '?'") + parser.add_argument("--azure_sas_key", help="Azure storage sas key, starts with '?'") parser.add_argument("--use_brainslice", action="store_true", help="Build with brain slice") - parser.add_argument("--brain_slice_package_path", help="Path to brain slice pacakges") - parser.add_argument("--brain_slice_package_name", help="Name of brain slice pakcages") + parser.add_argument("--brain_slice_package_path", help="Path to brain slice packages") + parser.add_argument("--brain_slice_package_name", help="Name of brain slice packages") parser.add_argument("--brain_slice_client_package_name", help="Name of brainslice client package") parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar") return parser.parse_args() +def resolve_executable_path(command_or_path): + """Returns the absolute path of an executable.""" + executable_path = shutil.which(command_or_path) + if executable_path is None: + raise BuildError("Failed to resolve executable path for '{}'.".format(command_or_path)) + return os.path.realpath(executable_path) + def is_windows(): return sys.platform.startswith("win") @@ -151,11 +163,10 @@ def install_apt_package(package): if is_sudo(): run_subprocess(['apt-get', 'install', '-y', package]) else: - log.error(package + " APT package missing. Please re-run this script using sudo to install.") - sys.exit(-1) + raise BuildError(package + " APT package missing. Please re-run this script using sudo to install.") def install_ubuntu_deps(args): - 'Check if the necessary Ubuntu dependencies are installed. Not required on docker. Provider help output if missing.' + 'Check if the necessary Ubuntu dependencies are installed. Not required on docker. Provide help output if missing.' # check we need the packages first if not (args.enable_pybind or args.use_openblas): @@ -171,8 +182,7 @@ def install_ubuntu_deps(args): install_apt_package("libopenblas-dev") except Exception as e: - log.error("Error setting up required APT packages. {}".format(str(e))) - sys.exit(-1) + raise BuildError("Error setting up required APT packages. {}".format(str(e))) def install_python_deps(): dep_packages = ['setuptools', 'wheel', 'numpy'] @@ -355,17 +365,15 @@ def setup_cuda_vars(args): cudnn_home_valid = (cudnn_home != None and os.path.exists(cudnn_home)) if (not cuda_home_valid or not cudnn_home_valid): - log.error("cuda_home and cudnn_home paths must be specified and valid.") - log.error("cuda_home='{}' valid={}. cudnn_home='{}' valid={}" - .format(cuda_home, cuda_home_valid, cudnn_home, cudnn_home_valid)) - sys.exit(-1) + raise BuildError("cuda_home and cudnn_home paths must be specified and valid.", + "cuda_home='{}' valid={}. cudnn_home='{}' valid={}" + .format(cuda_home, cuda_home_valid, cudnn_home, cudnn_home_valid)) if (is_windows()): # Validate that the cudnn_home is pointing at the right level if (not os.path.exists(os.path.join(cudnn_home, "bin"))): - log.error("cudnn_home path should include the 'cuda' folder, and must contain the CUDNN 'bin' directory.") - log.error("cudnn_home='{}'".format(cudnn_home)) - sys.exit(-1) + raise BuildError("cudnn_home path should include the 'cuda' folder, and must contain the CUDNN 'bin' directory.", + "cudnn_home='{}'".format(cudnn_home)) os.environ["CUDA_PATH"] = cuda_home os.environ["CUDA_TOOLKIT_ROOT_DIR"] = cuda_home @@ -377,8 +385,7 @@ def setup_cuda_vars(args): # Add version specific CUDA_PATH_Vx_y value as the Visual Studio build files require that version_file = os.path.join(cuda_home, 'version.txt') if not os.path.exists(version_file): - log.error("No version file found in CUDA install directory. Looked for " + version_file) - sys.exit(-1) + raise BuildError("No version file found in CUDA install directory. Looked for " + version_file) cuda_major_version = "unknown" @@ -387,8 +394,7 @@ def setup_cuda_vars(args): first_line = f.readline() m = re.match("CUDA Version (\d+).(\d+)", first_line) if not m: - log.error("Couldn't read version from first line of " + version_file) - sys.exit(-1) + raise BuildError("Couldn't read version from first line of " + version_file) cuda_major_version = m.group(1) minor = m.group(2) @@ -403,11 +409,10 @@ def setup_cuda_vars(args): log.warning("See build.md in the root ONNXRuntime directory for instructions on installing the Visual C++ 2017 14.11 toolset if needed.") elif cuda_major_version == "9" and vc_ver[0] == "14" and int(vc_ver[1]) > 11: - log.error("Visual C++ Tools version not supported by CUDA v9. You must setup the environment to use the 14.11 toolset.") - log.info("Current version is {}. CUDA 9.2 requires version 14.11.*".format(vc_ver_str)) - log.info("If necessary manually install the 14.11 toolset using the Visual Studio 2017 updater.") - log.info("See 'Windows CUDA Build' in build.md in the root directory of this repository.") - sys.exit(-1) + raise BuildError("Visual C++ Tools version not supported by CUDA v9. You must setup the environment to use the 14.11 toolset.", + "Current version is {}. CUDA 9.2 requires version 14.11.*".format(vc_ver_str), + "If necessary manually install the 14.11 toolset using the Visual Studio 2017 updater.", + "See 'Windows CUDA Build' in build.md in the root directory of this repository.") return cuda_home, cudnn_home @@ -480,7 +485,6 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda): def main(): args = parse_arguments() - cmake_path = args.cmake_path cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else [] # if there was no explicit argument saying what to do, default to update, build and test. @@ -496,7 +500,8 @@ def main(): configs = set(args.config) # setup paths and directories - ctest_path = args.ctest_path + cmake_path = resolve_executable_path(args.cmake_path) + ctest_path = resolve_executable_path(args.ctest_path) build_dir = args.build_dir script_dir = os.path.realpath(os.path.dirname(__file__)) source_dir = os.path.normpath(os.path.join(script_dir, "..", "..")) @@ -563,4 +568,8 @@ def main(): log.info("Build complete") if __name__ == "__main__": - sys.exit(main()) + try: + sys.exit(main()) + except BuildError as e: + log.error(str(e)) + sys.exit(1) diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 52a3e6a0004ec..f9fb03e3dca0d 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -5,7 +5,7 @@ jobs: - task: BatchScript@1 inputs: filename: build.bat - arguments: ' --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests' + arguments: ' --config Debug Release --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests' workingFolder: "$(Build.SourcesDirectory)" - task: CmdLine@1 diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 53c1f1a9baa14..cd6beffae8354 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -18,7 +18,7 @@ jobs: - task: BatchScript@1 inputs: filename: build.bat - arguments: ' --enable_onnx_tests --use_mkldnn --build_shared_lib --build_csharp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' + arguments: ' --config Debug Release --enable_onnx_tests --use_mkldnn --build_shared_lib --build_csharp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' workingFolder: "$(Build.SourcesDirectory)" - task: PowerShell@1 displayName: 'Clean up CUDA props files' diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 364cbb8bf8d19..d092283f78010 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -33,8 +33,8 @@ else #Install ONNX #5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3 #bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0 - #0a4d5abdf4939ab0842a5eadcc16a3bf0738f901 is v1.3.0 latest - for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "0a4d5abdf4939ab0842a5eadcc16a3bf0738f901"; do + #0c8d857bb162431912b255d5c0e773fb7c131a65 is v1.3.0 latest + for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "0c8d857bb162431912b255d5c0e773fb7c131a65"; do if [ -z ${lastest_onnx_version+x} ]; then echo "first pass"; else