From 7aef8a1ccab9c0ba7f6f47f6c5237fc071d870b8 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Thu, 22 Nov 2018 20:56:43 -0800 Subject: [PATCH] Sync with internal master. --- cmake/onnxruntime.cmake | 4 + cmake/onnxruntime_common.cmake | 2 +- cmake/onnxruntime_framework.cmake | 2 +- cmake/onnxruntime_graph.cmake | 2 +- cmake/onnxruntime_providers.cmake | 6 +- cmake/onnxruntime_session.cmake | 2 +- cmake/onnxruntime_unittests.cmake | 23 +- .../Program.cs | 40 +-- .../ExecutionProviderFactory.cs | 76 ++++ .../InferenceSession.cs | 128 ++++++- .../NamedOnnxValue.cs | 49 +++ .../Microsoft.ML.OnnxRuntime/NativeMethods.cs | 66 ++-- .../NativeOnnxObjectHandle.cs | 28 ++ .../NativeOnnxTensorMemory.cs | 49 +-- .../SessionOptions.cs | 71 ++-- .../InferenceTest.cs | 32 +- docs/C_API.md | 31 +- .../onnxruntime/core/framework/data_types.h | 6 + .../onnxruntime/core/framework/onnx_object.h | 3 +- .../core/framework/onnx_object_cxx.h | 47 +++ .../onnxruntime/core/framework/run_options.h | 7 +- .../framework/tensor_type_and_shape_c_api.h | 99 +++++ include/onnxruntime/core/graph/graph_base.h | 49 +-- .../core/session/onnxruntime_c_api.h | 143 +++----- .../core/session/onnxruntime_cxx_api.h | 59 +-- .../session/tensor_type_and_shape_c_api.h | 62 ---- onnxruntime/contrib_ops/contrib_ops.cc | 51 ++- .../contrib_ops/cpu/quantize_linear.cc | 177 +++++++++ onnxruntime/contrib_ops/cpu/quantize_linear.h | 41 +++ onnxruntime/core/codegen/tvm/tvm_kernel.h | 4 +- .../core/framework/allocation_planner.cc | 34 +- .../core/framework/allocation_planner.h | 4 +- onnxruntime/core/framework/cblas.h | 8 +- onnxruntime/core/framework/onnx_object.cc | 9 + .../core/framework/onnxruntime_typeinfo.cc | 127 +++++++ .../core/framework/onnxruntime_typeinfo.h | 40 +++ onnxruntime/core/framework/run_options.cc | 20 -- .../framework/session_state_initializer.cc | 29 +- .../framework/session_state_initializer.h | 11 +- .../core/framework/tensor_type_and_shape.cc | 186 ++++++++++ onnxruntime/core/graph/graph.cc | 237 ++++++------ onnxruntime/core/mlas/lib/mlasi.h | 11 + onnxruntime/core/mlas/lib/pooling.cpp | 172 ++++++++- onnxruntime/core/mlas/lib/sgemm.cpp | 4 +- .../providers/cpu/math/element_wise_ops.cc | 309 ---------------- .../providers/cpu/math/element_wise_ops.h | 310 ++++++++++++++++ onnxruntime/core/providers/cpu/symbols.txt | 15 +- .../core/session/abi_session_options.cc | 23 +- .../core/session/abi_session_options_impl.h | 7 +- onnxruntime/core/session/inference_session.cc | 19 +- onnxruntime/core/session/onnxruntime_c_api.cc | 339 +++++++----------- .../core/session/tensor_type_and_shape.cc | 90 ----- .../test/contrib_ops/quantize_linear_test.cc | 80 +++++ .../test/framework/allocation_planner_test.cc | 28 +- .../test/framework/execution_frame_test.cc | 4 +- onnxruntime/test/onnx/main.cc | 4 +- onnxruntime/test/onnx/runner.cc | 4 +- onnxruntime/test/onnxruntime_exec/Runtime.h | 2 +- .../providers/cpu/controlflow/scan_test.cc | 12 +- .../shared_lib/fns_candy_style_transfer.c | 28 +- onnxruntime/test/shared_lib/test_fixture.h | 12 +- onnxruntime/test/shared_lib/test_inference.cc | 77 ++-- onnxruntime/test/shared_lib/test_io_types.cc | 45 +++ onnxruntime/test/util/compare_mlvalue.cc | 143 ++++---- .../test/util/include/test/compare_mlvalue.h | 3 +- 65 files changed, 2460 insertions(+), 1345 deletions(-) create mode 100644 csharp/src/Microsoft.ML.OnnxRuntime/ExecutionProviderFactory.cs create mode 100644 csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxObjectHandle.cs create mode 100644 include/onnxruntime/core/framework/onnx_object_cxx.h create mode 100644 include/onnxruntime/core/framework/tensor_type_and_shape_c_api.h delete mode 100644 include/onnxruntime/core/session/tensor_type_and_shape_c_api.h create mode 100644 onnxruntime/contrib_ops/cpu/quantize_linear.cc create mode 100644 onnxruntime/contrib_ops/cpu/quantize_linear.h create mode 100644 onnxruntime/core/framework/onnxruntime_typeinfo.cc create mode 100644 onnxruntime/core/framework/onnxruntime_typeinfo.h create mode 100644 onnxruntime/core/framework/tensor_type_and_shape.cc delete mode 100644 onnxruntime/core/session/tensor_type_and_shape.cc create mode 100644 onnxruntime/test/contrib_ops/quantize_linear_test.cc create mode 100644 onnxruntime/test/shared_lib/test_io_types.cc diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 901052ca7c451..db5c4468124b8 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -10,6 +10,10 @@ else() endif() +#If you want to verify if there is any extra line in symbols.txt, run +# nm -C -g --defined libonnxruntime.so |grep -v '\sA\s' | cut -f 3 -d ' ' | sort +# after build + list(APPEND SYMBOL_FILES "${REPO_ROOT}/tools/ci_build/gen_def.py") foreach(f ${ONNXRUNTIME_PROVIDER_NAMES}) list(APPEND SYMBOL_FILES "${ONNXRUNTIME_ROOT}/core/providers/${f}/symbols.txt") diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index eed99f87ef78c..3af8fd048405d 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -44,7 +44,7 @@ target_include_directories(onnxruntime_common PRIVATE ${ONNXRUNTIME_ROOT} ${date # logging uses date. threadpool uses eigen add_dependencies(onnxruntime_common date eigen gsl) -install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/common DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/common) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/common DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) set_target_properties(onnxruntime_common PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_common PROPERTIES FOLDER "ONNXRuntime") diff --git a/cmake/onnxruntime_framework.cmake b/cmake/onnxruntime_framework.cmake index b74cdc6a88d4f..04df731e919d8 100644 --- a/cmake/onnxruntime_framework.cmake +++ b/cmake/onnxruntime_framework.cmake @@ -18,7 +18,7 @@ set_target_properties(onnxruntime_framework PROPERTIES FOLDER "ONNXRuntime") # need onnx to build to create headers that this project includes add_dependencies(onnxruntime_framework ${onnxruntime_EXTERNAL_DEPENDENCIES}) -install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/framework DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/framework) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/framework DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) if (WIN32) # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. set_target_properties(onnxruntime_framework PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) diff --git a/cmake/onnxruntime_graph.cmake b/cmake/onnxruntime_graph.cmake index d83c07ca044f0..f01907a33be26 100644 --- a/cmake/onnxruntime_graph.cmake +++ b/cmake/onnxruntime_graph.cmake @@ -17,7 +17,7 @@ onnxruntime_add_include_to_target(onnxruntime_graph onnx protobuf::libprotobuf) target_include_directories(onnxruntime_graph PRIVATE ${ONNXRUNTIME_ROOT}) set_target_properties(onnxruntime_graph PROPERTIES FOLDER "ONNXRuntime") set_target_properties(onnxruntime_graph PROPERTIES LINKER_LANGUAGE CXX) -install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/graph DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/graph) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/graph DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_graph_src} ${onnxruntime_ir_defs_src}) if (WIN32) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 379738325a7f8..79734f8a1315d 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -34,7 +34,7 @@ add_library(onnxruntime_providers ${onnxruntime_providers_common_srcs} ${onnxrun onnxruntime_add_include_to_target(onnxruntime_providers onnx protobuf::libprotobuf) target_include_directories(onnxruntime_providers PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) add_dependencies(onnxruntime_providers eigen gsl onnx) -install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cpu DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers/cpu) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cpu DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers PROPERTIES FOLDER "ONNXRuntime") @@ -61,7 +61,7 @@ if (onnxruntime_USE_CUDA) onnxruntime_add_include_to_target(onnxruntime_providers_cuda onnx protobuf::libprotobuf) add_dependencies(onnxruntime_providers_cuda eigen ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies}) target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES}) - install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cuda DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers/cuda) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cuda DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_cuda PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(onnxruntime_providers_cuda PROPERTIES FOLDER "ONNXRuntime") if (WIN32) @@ -95,7 +95,7 @@ if (onnxruntime_USE_MKLDNN) add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime") target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) - install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/mkldnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers/mkldnn) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/mkldnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX) endif() diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake index 9cc92ec1f6f3f..1075ad33a565e 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -10,7 +10,7 @@ file(GLOB onnxruntime_session_srcs source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_session_srcs}) add_library(onnxruntime_session ${onnxruntime_session_srcs}) -install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/session DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/session) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/session DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) onnxruntime_add_include_to_target(onnxruntime_session onnx protobuf::libprotobuf) target_include_directories(onnxruntime_session PRIVATE ${ONNXRUNTIME_ROOT}) add_dependencies(onnxruntime_session ${onnxruntime_EXTERNAL_DEPENDENCIES}) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index f4c09cdd1d5fa..3108c249dd853 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -525,14 +525,17 @@ if (onnxruntime_BUILD_SHARED_LIB) # this program shouldn't have direct depedency on CUDA # CUDA is part of ${ONNX_DLL} set (ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR "${ONNXRUNTIME_ROOT}/test/shared_lib") - add_executable(onnxruntime_shared_lib_test - ${ONNXRUNTIME_ROOT}/test/util/test_allocator.cc - ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_fixture.h - ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc - ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_session_options.cc - ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_run_options.cc - ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_allocator.cc - ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc) + set (onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_ROOT}/test/util/test_allocator.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_fixture.h + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_session_options.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_run_options.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_allocator.cc + ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_inference.cc) + if(onnxruntime_RUN_ONNX_TESTS) + list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_io_types.cc) + endif() + add_executable(onnxruntime_shared_lib_test ${onnxruntime_shared_lib_test_SRC}) onnxruntime_add_include_to_target(onnxruntime_shared_lib_test onnxruntime_test_utils) target_include_directories(onnxruntime_shared_lib_test PRIVATE "${TEST_SRC_DIR}/util/include" "${PROJECT_SOURCE_DIR}/include") if(WIN32) @@ -557,7 +560,3 @@ add_executable(onnxruntime_mlas_test ${TEST_SRC_DIR}/mlas/unittest.cpp) target_include_directories(onnxruntime_mlas_test PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc) target_link_libraries(onnxruntime_mlas_test PRIVATE onnxruntime_mlas) set_target_properties(onnxruntime_mlas_test PROPERTIES FOLDER "ONNXRuntimeTest") - -if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) - include(onnxruntime_standalone_tests_internal.cmake) -endif() diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs index b5e89074a66a7..c92a366b534a9 100644 --- a/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs @@ -28,16 +28,15 @@ static void UseApi() using (var session = new InferenceSession(modelPath)) { var inputMeta = session.InputMetadata; + var container = new List(); - // User should be able to detect input name/type/shape from the metadata. - // Currently InputMetadata implementation is inclomplete, so assuming Tensor of predefined dimension. - - var shape0 = new int[] { 1, 3, 224, 224 }; - float[] inputData0 = LoadInputsFloat(); - var tensor = new DenseTensor(inputData0, shape0); + float[] inputData = LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model - var container = new List(); - container.Add(new NamedOnnxValue("data_0", tensor)); + foreach (var name in inputMeta.Keys) + { + var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions); + container.Add(new NamedOnnxValue(name, tensor)); + } // Run the inference var results = session.Run(container); // results is an IReadOnlyList container @@ -49,40 +48,27 @@ static void UseApi() Console.WriteLine(r.AsTensor().GetArrayString()); } - // Just try some GC collect - results = null; - container = null; - - GC.Collect(); - GC.WaitForPendingFinalizers(); } } - static int[] LoadInputsInt32() + static float[] LoadTensorFromFile(string filename) { - return null; - } - - static float[] LoadInputsFloat() - { - // input: data_0 = float32[1,3,224,224] for squeezenet model - // output: softmaxout_1 = float32[1,1000,1,1] - uint size = 1 * 3 * 224 * 224; - float[] tensor = new float[size]; + var tensorData = new List(); // read data from file - using (var inputFile = new System.IO.StreamReader(@"bench.in")) + using (var inputFile = new System.IO.StreamReader(filename)) { inputFile.ReadLine(); //skip the input name string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < dataStr.Length; i++) { - tensor[i] = Single.Parse(dataStr[i]); + tensorData.Add(Single.Parse(dataStr[i])); } } - return tensor; + return tensorData.ToArray(); } + } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/ExecutionProviderFactory.cs b/csharp/src/Microsoft.ML.OnnxRuntime/ExecutionProviderFactory.cs new file mode 100644 index 0000000000000..409076fb6d5cd --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/ExecutionProviderFactory.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + + internal class CpuExecutionProviderFactory: NativeOnnxObjectHandle + { + protected static readonly Lazy _default = new Lazy(() => new CpuExecutionProviderFactory()); + + public CpuExecutionProviderFactory(bool useArena=true) + :base(IntPtr.Zero) + { + int useArenaInt = useArena ? 1 : 0; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeCreateCpuExecutionProviderFactory(useArenaInt, out handle)); + } + catch(OnnxRuntimeException e) + { + if (IsInvalid) + { + ReleaseHandle(); + handle = IntPtr.Zero; + } + throw e; + } + } + + public static CpuExecutionProviderFactory Default + { + get + { + return _default.Value; + } + } + } + + internal class MklDnnExecutionProviderFactory : NativeOnnxObjectHandle + { + protected static readonly Lazy _default = new Lazy(() => new MklDnnExecutionProviderFactory()); + + public MklDnnExecutionProviderFactory(bool useArena = true) + :base(IntPtr.Zero) + { + int useArenaInt = useArena ? 1 : 0; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeCreateMkldnnExecutionProviderFactory(useArenaInt, out handle)); + } + catch (OnnxRuntimeException e) + { + if (IsInvalid) + { + ReleaseHandle(); + handle = IntPtr.Zero; + } + throw e; + } + } + + public static MklDnnExecutionProviderFactory Default + { + get + { + return _default.Value; + } + } + } + + + + +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index ae4e5ad309948..882fe47994e9d 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -11,13 +11,9 @@ namespace Microsoft.ML.OnnxRuntime { - public struct RunOptions - { - // placeholder for RunOptions - } /// - /// Represents an Inference Session against an ONNX Model + /// Represents an Inference Session on an ONNX Model /// public class InferenceSession: IDisposable { @@ -56,7 +52,7 @@ public InferenceSession(string modelPath, SessionOptions options) // get all the output names for (ulong i = 0; i < inputCount; i++) { - _inputMetadata[GetInputName(i)] = new NodeMetadata(); //TODO: fill the shape/type when C-api available + _inputMetadata[GetInputName(i)] = GetInputMetadata(i); } // get output count @@ -66,7 +62,7 @@ public InferenceSession(string modelPath, SessionOptions options) // get all the output names for (ulong i = 0; i < outputCount; i++) { - _outputMetadata[GetOutputName(i)] = new NodeMetadata(); //TODO: fill the shape/type when C-api available + _outputMetadata[GetOutputName(i)] = GetOutputMetadata(i); } } catch (OnnxRuntimeException e) @@ -104,7 +100,12 @@ public ModelMetadata ModelMetadata } } - public IReadOnlyCollection Run(IReadOnlyCollection inputs, RunOptions options = new RunOptions()) + public IReadOnlyCollection Run(IReadOnlyCollection inputs) + { + return Run(inputs, RunOptions.Default); + } + + public IReadOnlyCollection Run(IReadOnlyCollection inputs, RunOptions options) { string[] outputNames = new string[_outputMetadata.Count]; _outputMetadata.Keys.CopyTo(outputNames, 0); @@ -118,7 +119,7 @@ public ModelMetadata ModelMetadata /// /// /// Output Tensors in a Dictionary - public IReadOnlyCollection Run(IReadOnlyCollection inputs, IReadOnlyCollection outputNames, RunOptions options = new RunOptions()) + public IReadOnlyCollection Run(IReadOnlyCollection inputs, IReadOnlyCollection outputNames, RunOptions options) { var inputNames = new string[inputs.Count]; var inputTensors = new IntPtr[inputs.Count]; @@ -129,7 +130,7 @@ public ModelMetadata ModelMetadata { inputNames[offset] = input.Name; - // create Tensor fromt the input if feasible, else throw notsupported exception for now + // create Tensor from the input if feasible, else throw notsupported exception for now input.ToNativeOnnxValue(out inputTensors[offset], out pinnedBufferHandles[offset]); offset++; @@ -140,6 +141,8 @@ public ModelMetadata ModelMetadata IntPtr status = NativeMethods.ONNXRuntimeRunInference( this._nativeHandle, + IntPtr.Zero, // TODO: use Run options when Run options creation API is available + // Passing null uses the default run options in the C-api inputNames, inputTensors, (ulong)(inputTensors.Length), /* TODO: size_t, make it portable for x86 arm */ @@ -239,9 +242,62 @@ private string GetInputName(ulong index) return str; } - #endregion + private NodeMetadata GetInputMetadata(ulong index) + { + IntPtr typeInfo = IntPtr.Zero; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeInferenceSessionGetInputTypeInfo(_nativeHandle, index, out typeInfo)); + return GetMetadataFromTypeInfo(typeInfo); + } + finally + { + if (typeInfo != IntPtr.Zero) + { + NativeMethods.ONNXRuntimeReleaseObject(typeInfo); + } + } + } + private NodeMetadata GetOutputMetadata(ulong index) + { + IntPtr typeInfo = IntPtr.Zero; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeInferenceSessionGetOutputTypeInfo(_nativeHandle, index, out typeInfo)); + return GetMetadataFromTypeInfo(typeInfo); + } + finally + { + if (typeInfo != IntPtr.Zero) + { + NativeMethods.ONNXRuntimeReleaseObject(typeInfo); + } + } + } + + private NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) + { + IntPtr tensorInfo = NativeMethods.ONNXRuntimeCastTypeInfoToTensorInfo(typeInfo); + // Convert the newly introduced ONNXRuntimeTypeInfo* to the older ONNXRuntimeTypeAndShapeInfo* + + TensorElementType type = NativeMethods.ONNXRuntimeGetTensorElementType(tensorInfo); + Type dotnetType = null; + int width = 0; + TensorElementTypeConverter.GetTypeAndWidth(type, out dotnetType, out width); + ulong numDimensions = NativeMethods.ONNXRuntimeGetNumOfDimensions(tensorInfo); + long[] dimensions = new long[(int)numDimensions]; + NativeMethods.ONNXRuntimeGetDimensions(tensorInfo, dimensions, numDimensions); + int[] intDimensions = new int[(int)numDimensions]; + for (ulong i = 0; i < numDimensions; i++) + { + intDimensions[i] = (int)dimensions[i]; + } + return new NodeMetadata(intDimensions, dotnetType); + } + + #endregion #region destructors disposers @@ -275,23 +331,61 @@ protected virtual void Dispose(bool disposing) } - public struct NodeMetadata + + /// + /// Resembles type and shape information of session-graph nodes, used for communicating the shape/type of input/output nodes + /// + public class NodeMetadata { - //TODO: currently shape and type is not available in C-api, so this struct may change based on implementation - public uint[] Shape + private int[] _dimensions; + private Type _type; + + internal NodeMetadata(int[] dimensions, Type type) { - get; internal set; + _dimensions = dimensions; + _type = type; + } + + public int[] Dimensions + { + get + { + return _dimensions; + } } public System.Type Type { - get; internal set; + get + { + return _type; + } } } - public struct ModelMetadata + public class ModelMetadata { //TODO: placeholder for Model metadata. Currently C-API does not expose this } + /// Sets various runtime options. + /// TODO: currently uses Default options only + public class RunOptions + { + protected static readonly Lazy _default = new Lazy(() => new RunOptions()); + + public static RunOptions Default + { + get + { + return _default.Value; + } + } + + private void RuntOptions() + { + + } + } + } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs index 829b63cdbcdec..9d02e664d3cd6 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs @@ -320,4 +320,53 @@ internal enum TensorElementType DataTypeMax = 17 } + internal static class TensorElementTypeConverter + { + public static void GetTypeAndWidth(TensorElementType elemType, out Type type, out int width) + { + switch (elemType) + { + case TensorElementType.Float: + type = typeof(float); + width = sizeof(float); + break; + case TensorElementType.Double: + type = typeof(double); + width = sizeof(double); + break; + case TensorElementType.Int16: + type = typeof(short); + width = sizeof(short); + break; + case TensorElementType.UInt16: + type = typeof(ushort); + width = sizeof(ushort); + break; + case TensorElementType.Int32: + type = typeof(int); + width = sizeof(int); + break; + case TensorElementType.UInt32: + type = typeof(uint); + width = sizeof(uint); + break; + case TensorElementType.Int64: + type = typeof(long); + width = sizeof(long); + break; + case TensorElementType.UInt64: + type = typeof(ulong); + width = sizeof(ulong); + break; + case TensorElementType.UInt8: + type = typeof(byte); + width = sizeof(byte); + break; + default: + type = null; + width = 0; + break; + } + } + } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 2eeac4da6c0ed..cb089b7bb32f6 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -49,20 +49,10 @@ internal static class NativeMethods IntPtr /* (ONNXRuntimeSessionOptions*) */sessopnOptions, out IntPtr /**/ session); - - [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInferenceAndFetchAll( - IntPtr /*(ONNXSessionPtr)*/ session, - string[] inputNames, - IntPtr[] /*(ONNXValuePtr[])*/ inputValues, - ulong inputLength, // size_t, TODO: make it portable for x86, arm - out IntPtr /* (ONNXValueListPtr*)*/ outputValues, - out ulong /* (size_t*) */ outputLength); //TODO: make it portable for x86, arm - - [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInference( IntPtr /*(ONNXSession*)*/ session, + IntPtr /*(ONNXSessionRunOptions*)*/ runOptions, // can be null to use the default options string[] inputNames, IntPtr[] /* (ONNXValue*[])*/ inputValues, ulong inputCount, /* TODO: size_t, make it portable for x86 arm */ @@ -99,6 +89,20 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca IntPtr /*(ONNXRuntimeAllocator*)*/ allocator, out IntPtr /*(char**)*/name); + // release the typeinfo using ONNXRuntimeReleaseObject + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetInputTypeInfo( + IntPtr /*(const ONNXSession*)*/ session, + ulong index, //TODO: port for size_t + out IntPtr /*(struct ONNXRuntimeTypeInfo**)*/ typeInfo); + + // release the typeinfo using ONNXRuntimeReleaseObject + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetOutputTypeInfo( + IntPtr /*(const ONNXSession*)*/ session, + ulong index, //TODO: port for size_t + out IntPtr /* (struct ONNXRuntimeTypeInfo**)*/ typeInfo); + [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXSession(IntPtr /*(ONNXSession*)*/session); @@ -106,18 +110,20 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca #endregion InferenceSession API #region SessionOptions API + + //Release using ONNXRuntimeReleaseObject [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*ONNXRuntimeSessionOptions* */ ONNXRuntimeCreateSessionOptions(); - //DEFINE_RUNTIME_CLASS(ONNXRuntimeSessionOptions) + [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXRuntimeSessionOptions(IntPtr /*(ONNXRuntimeSessionOptions*)*/ sessionOptions); + public static extern IntPtr /*(ONNXRuntimeSessionOptions*)*/ONNXRuntimeCloneSessionOptions(IntPtr /*(ONNXRuntimeSessionOptions*)*/ sessionOptions); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ONNXRuntimeEnableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options); + public static extern void ONNXRuntimeEnableSequentialExecution(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ONNXRuntimeDisableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options); + public static extern void ONNXRuntimeDisableSequentialExecution(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeEnableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options, string profilePathPrefix); @@ -146,17 +152,28 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca [DllImport(nativeLib, CharSet = charSet)] public static extern int ONNXRuntimeSetSessionThreadPoolSize(IntPtr /* ONNXRuntimeSessionOptions* */ options, int sessionThreadPoolSize); + ///** + // * The order of invocation indicates the preference order as well. In other words call this method + // * on your most preferred execution provider first followed by the less preferred ones. + // * Calling this API is optional in which case onnxruntime will use its internal CPU execution provider. + // */ [DllImport(nativeLib, CharSet = charSet)] - public static extern int ONNXRuntimeEnableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options, int deviceId); + public static extern void ONNXRuntimeSessionOptionsAppendExecutionProvider(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options, IntPtr /* (ONNXRuntimeProviderFactoryPtr*)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ONNXRuntimeDisableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateCpuExecutionProviderFactory(int use_arena, out IntPtr /*(ONNXRuntimeProviderFactoryPtr*)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern int ONNXRuntimeEnableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateMkldnnExecutionProviderFactory(int use_arena, out IntPtr /*(ONNXRuntimeProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ONNXRuntimeDisableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options); + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateCUDAExecutionProviderFactory(int device_id, out IntPtr /*(ONNXRuntimeProviderFactoryPtr**)*/ factory); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateNupharExecutionProviderFactory(int device_id, string target_str, out IntPtr /*(ONNXRuntimeProviderFactoryPtr**)*/ factory); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void ONNXRuntimeAddCustomOp(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options, string custom_op_path); #endregion @@ -246,6 +263,10 @@ public enum MemoryType //[DllImport(nativeLib, CharSet = charSet)] //public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count); + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ + ONNXRuntimeCastTypeInfoToTensorInfo(IntPtr /*(struct ONNXRuntimeTypeInfo*)*/ typeInfo); + [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeAndType(IntPtr /*(ONNXValue*)*/ value, out IntPtr /*(struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); @@ -273,16 +294,9 @@ public static extern void ONNXRuntimeGetDimensions( [DllImport(nativeLib, CharSet = charSet)] public static extern long ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); - - [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXValuePtr)*/ ONNXRuntimeONNXValueListGetNthValue(IntPtr /*(ONNXValueListPtr)*/ list, ulong index); // 0-based index TODO: size_t, make it portable for x86, arm - [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value); - [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXValueList(IntPtr /*(ONNXValueList*)*/ valueList); - #endregion } //class NativeMethods } //namespace diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxObjectHandle.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxObjectHandle.cs new file mode 100644 index 0000000000000..f30ded50c7a8c --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxObjectHandle.cs @@ -0,0 +1,28 @@ +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + + internal class NativeOnnxObjectHandle : SafeHandle + { + public NativeOnnxObjectHandle(IntPtr ptr) + : base(IntPtr.Zero, true) + { + handle = ptr; + } + public override bool IsInvalid + { + get + { + return (handle == IntPtr.Zero); + } + } + + protected override bool ReleaseHandle() + { + NativeMethods.ONNXRuntimeReleaseObject(handle); + return true; + } + } +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs index b5f66e94658b2..f70aa8083eec7 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs @@ -33,7 +33,7 @@ public NativeOnnxTensorMemory(IntPtr onnxValueHandle) Type type = null; int width = 0; - GetTypeAndWidth(elemType, out type, out width); + TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); if (typeof(T) != type) throw new NotSupportedException(nameof(NativeOnnxTensorMemory)+" does not support T = "+nameof(T)); _elementWidth = width; @@ -182,52 +182,7 @@ protected override bool TryGetArray(out ArraySegment arraySegment) } - internal static void GetTypeAndWidth(TensorElementType elemType, out Type type, out int width) - { - switch (elemType) - { - case TensorElementType.Float: - type = typeof(float); - width = sizeof(float); - break; - case TensorElementType.Double: - type = typeof(double); - width = sizeof(double); - break; - case TensorElementType.Int16: - type = typeof(short); - width = sizeof(short); - break; - case TensorElementType.UInt16: - type = typeof(ushort); - width = sizeof(ushort); - break; - case TensorElementType.Int32: - type = typeof(int); - width = sizeof(int); - break; - case TensorElementType.UInt32: - type = typeof(uint); - width = sizeof(uint); - break; - case TensorElementType.Int64: - type = typeof(long); - width = sizeof(long); - break; - case TensorElementType.UInt64: - type = typeof(ulong); - width = sizeof(ulong); - break; - case TensorElementType.UInt8: - type = typeof(byte); - width = sizeof(byte); - break; - default: - type = null; - width = 0; - break; - } - } + } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index 16daea4cf1b54..98f90d9e401cb 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -2,58 +2,83 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; -using System.Text; +using System.Runtime.InteropServices; + namespace Microsoft.ML.OnnxRuntime { - public class SessionOptions : IDisposable + public enum ExecutionProvider + { + Cpu, + MklDnn + //TODO: add more providers gradually + }; + + public class SessionOptions { - private static SessionOptions _defaultOptions = new SessionOptions(); - private IntPtr _nativeHandle; + protected SafeHandle _nativeOption; + protected static readonly Lazy _default = new Lazy(MakeSessionOptionWithMklDnnProvider); public SessionOptions() { - _nativeHandle = NativeMethods.ONNXRuntimeCreateSessionOptions(); + _nativeOption = new NativeOnnxObjectHandle(NativeMethods.ONNXRuntimeCreateSessionOptions()); } - internal IntPtr NativeHandle + public static SessionOptions Default { get { - return _nativeHandle; + return _default.Value; } } - public static SessionOptions Default + public void AppendExecutionProvider(ExecutionProvider provider) { - get + switch (provider) { - return _defaultOptions; + case ExecutionProvider.Cpu: + AppendExecutionProvider(CpuExecutionProviderFactory.Default); + break; + case ExecutionProvider.MklDnn: + AppendExecutionProvider(MklDnnExecutionProviderFactory.Default); + break; + default: + break; } } - #region destructors disposers - ~SessionOptions() + + private static SessionOptions MakeSessionOptionWithMklDnnProvider() { - Dispose(false); + SessionOptions options = new SessionOptions(); + options.AppendExecutionProvider(MklDnnExecutionProviderFactory.Default); + options.AppendExecutionProvider(CpuExecutionProviderFactory.Default); + + return options; } - public void Dispose() + + internal IntPtr NativeHandle { - GC.SuppressFinalize(this); - Dispose(true); + get + { + return _nativeOption.DangerousGetHandle(); //Note: this is unsafe, and not ref counted, use with caution + } } - protected virtual void Dispose(bool disposing) + private void AppendExecutionProvider(NativeOnnxObjectHandle providerFactory) { - if (disposing) + unsafe { - // cleanup managed resources - } + bool success = false; + providerFactory.DangerousAddRef(ref success); + if (success) + { + NativeMethods.ONNXRuntimeSessionOptionsAppendExecutionProvider(_nativeOption.DangerousGetHandle(), providerFactory.DangerousGetHandle()); + providerFactory.DangerousRelease(); + } - // cleanup unmanaged resources + } } - #endregion } } diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 40e655ee9a45a..5c3ec76fd3e6d 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -25,12 +25,24 @@ public void CanCreateAndDisposeSessionWithModelPath() Assert.NotNull(session.InputMetadata); Assert.Equal(1, session.InputMetadata.Count); // 1 input node Assert.True(session.InputMetadata.ContainsKey("data_0")); // input node name + Assert.Equal(typeof(float), session.InputMetadata["data_0"].Type); + var expectedInputDimensions = new int[] { 1, 3, 224, 224 }; + Assert.Equal(expectedInputDimensions.Length, session.InputMetadata["data_0"].Dimensions.Length); + for (int i = 0; i < expectedInputDimensions.Length; i++) + { + Assert.Equal(expectedInputDimensions[i], session.InputMetadata["data_0"].Dimensions[i]); + } Assert.NotNull(session.OutputMetadata); Assert.Equal(1, session.OutputMetadata.Count); // 1 output node Assert.True(session.OutputMetadata.ContainsKey("softmaxout_1")); // output node name - - //TODO: verify shape/type of the input/output nodes when API available + Assert.Equal(typeof(float), session.OutputMetadata["softmaxout_1"].Type); + var expectedOutputDimensions = new int[] { 1, 1000, 1, 1 }; + Assert.Equal(expectedOutputDimensions.Length, session.OutputMetadata["softmaxout_1"].Dimensions.Length); + for (int i = 0; i < expectedOutputDimensions.Length; i++) + { + Assert.Equal(expectedOutputDimensions[i], session.OutputMetadata["softmaxout_1"].Dimensions[i]); + } } } @@ -42,16 +54,16 @@ private void CanRunInferenceOnAModel() using (var session = new InferenceSession(modelPath)) { var inputMeta = session.InputMetadata; + var container = new List(); - // User should be able to detect input name/type/shape from the metadata. - // Currently InputMetadata implementation is inclomplete, so assuming Tensor of predefined dimension. - - var shape0 = new int[] { 1, 3, 224, 224 }; - float[] inputData0 = LoadTensorFromFile(@"bench.in"); - var tensor = new DenseTensor(inputData0, shape0); + float[] inputData = LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model - var container = new List(); - container.Add(new NamedOnnxValue("data_0", tensor)); + foreach (var name in inputMeta.Keys) + { + Assert.Equal(typeof(float), inputMeta[name].Type); + var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions); + container.Add(new NamedOnnxValue(name, tensor)); + } // Run the inference var results = session.Run(container); // results is an IReadOnlyList container diff --git a/docs/C_API.md b/docs/C_API.md index 7cf5737bccfbb..7661b905194a7 100644 --- a/docs/C_API.md +++ b/docs/C_API.md @@ -1,9 +1,20 @@ # C API -## Headers -[onnxruntime_c_api.h](include/onnxruntime/core/session/onnxruntime_c_api.h) +# Q: Why having a C API? +Q: Why not just live in C++ world? Why must C? +A: We want to distribute onnxruntime as a DLL, which can be used in .Net languages through [P/Invoke](https://docs.microsoft.com/en-us/cpp/dotnet/how-to-call-native-dlls-from-managed-code-using-pinvoke). +Then this is the only option we have. -## Functionality +Q: Is it only for .Net? +A: No. It is designed for +1. Creating language bindings for onnxruntime.e.g. C#, python, java, ... +2. Dynamic linking always has some benefits. For example, for solving diamond dependency problem. + +Q: Can I export C++ types and functions across DLL or "Shared Object" Library(.so) boundaries? +A: Well, you can, but it's not a good practice. And we won't do it in this project. + + +## What's inside * Creating an InferenceSession from an on-disk model file and a set of SessionOptions. * Registering customized loggers. * Registering customized allocators. @@ -12,3 +23,17 @@ * Converting an in-memory ONNX Tensor encoded in protobuf format, to a pointer that can be used as model input. * Setting the thread pool size for each session. * Dynamically loading custom ops. + +## How to use it + +Include [onnxruntime_c_api.h](include/onnxruntime/core/session/onnxruntime_c_api.h) in your source code. + +Then, +1. Call ONNXRuntimeInitialize +2. Create Session: ONNXRuntimeCreateInferenceSession(env, model_uri, nullptr,...) +3. Create Tensor + 1) ONNXRuntimeCreateAllocatorInfo + 2) ONNXRuntimeCreateTensorWithDataAsONNXValue +4. ONNXRuntimeRunInference + + diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h index 487fec6d8a8ed..baf435106f6d2 100644 --- a/include/onnxruntime/core/framework/data_types.h +++ b/include/onnxruntime/core/framework/data_types.h @@ -107,6 +107,12 @@ class DataTypeImpl { template static MLDataType GetTensorType(); + /** + * Convert an ONNX TypeProto to onnxruntime DataTypeImpl. + * However, this conversion is lossy. Don't try to use 'this->GetTypeProto()' converting it back + * Don't pass the returned value to MLValue::MLValue(...) function + * \param proto + */ static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto); // Registers ONNX_NAMESPACE::DataType (internalized string) with diff --git a/include/onnxruntime/core/framework/onnx_object.h b/include/onnxruntime/core/framework/onnx_object.h index e4989eda8176b..0ef52203faeca 100644 --- a/include/onnxruntime/core/framework/onnx_object.h +++ b/include/onnxruntime/core/framework/onnx_object.h @@ -28,13 +28,14 @@ typedef struct ONNXObject { ONNXRUNTIME_API(uint32_t, ONNXRuntimeAddRefToObject, _In_ void* ptr); /** - * + * * A wrapper to "(*(ONNXObject**)ptr)->Release(ptr)" * WARNING: There is NO type checking in this function. * \param ptr Can be NULL. If it's NULL, this function will return zero. * \return the new reference count. */ ONNXRUNTIME_API(uint32_t, ONNXRuntimeReleaseObject, _Inout_opt_ void* ptr); + #ifdef __cplusplus } #endif diff --git a/include/onnxruntime/core/framework/onnx_object_cxx.h b/include/onnxruntime/core/framework/onnx_object_cxx.h new file mode 100644 index 0000000000000..c9e25ea840e7a --- /dev/null +++ b/include/onnxruntime/core/framework/onnx_object_cxx.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/onnx_object.h" +#include + +namespace onnxruntime { + +/** + * Even it's designed to be inherited, this class doesn't have a virtual destructor. + * No vtable is allowed in this class and its subclasses. + * \tparam T subclass type name + */ +template +class ObjectBase { + private: + static ONNXObject static_cls; + + protected: + const ONNXObject* const ONNXRUNTIME_ATTRIBUTE_UNUSED cls_; + std::atomic_int ref_count; + ObjectBase() : cls_(&static_cls), ref_count(1) { + } + + static uint32_t ONNXRUNTIME_API_STATUSCALL ONNXRuntimeReleaseImpl(void* this_) { + T* this_ptr = reinterpret_cast(this_); + if (--this_ptr->ref_count == 0) + delete this_ptr; + return 0; + } + + static uint32_t ONNXRUNTIME_API_STATUSCALL ONNXRuntimeAddRefImpl(void* this_) { + T* this_ptr = reinterpret_cast(this_); + ++this_ptr->ref_count; + return 0; + } +}; + +template +ONNXObject ObjectBase::static_cls = {ObjectBase::ONNXRuntimeAddRefImpl, ObjectBase::ONNXRuntimeReleaseImpl}; + +} // namespace onnxruntime + +#define ONNXRUNTIME_CHECK_C_OBJECT_LAYOUT \ + { assert((char*)&ref_count == (char*)this + sizeof(this)); } diff --git a/include/onnxruntime/core/framework/run_options.h b/include/onnxruntime/core/framework/run_options.h index 25268d56356d3..e3b8a0b178741 100644 --- a/include/onnxruntime/core/framework/run_options.h +++ b/include/onnxruntime/core/framework/run_options.h @@ -7,20 +7,19 @@ #include #include #include "core/framework/onnx_object.h" +#include "core/framework/onnx_object_cxx.h" /** * Configuration information for a single Run. */ -struct ONNXRuntimeRunOptions { - const ONNXObject* const cls; - std::atomic_int ref_count; +struct ONNXRuntimeRunOptions : public onnxruntime::ObjectBase { unsigned run_log_verbosity_level = 0; ///< applies to a particular Run() invocation std::string run_tag; ///< to identify logs generated by a particular Run() invocation /// set to 'true' to terminate any currently executing Run() calls that are using this /// ONNXRuntimeRunOptions instance. the individual calls will exit gracefully and return an error status. bool terminate = false; - ONNXRuntimeRunOptions(); + ONNXRuntimeRunOptions() = default; ~ONNXRuntimeRunOptions() = default; // disable copy, move and assignment. we don't want accidental copies, to ensure that the instance provided to diff --git a/include/onnxruntime/core/framework/tensor_type_and_shape_c_api.h b/include/onnxruntime/core/framework/tensor_type_and_shape_c_api.h new file mode 100644 index 0000000000000..e38b2c7817749 --- /dev/null +++ b/include/onnxruntime/core/framework/tensor_type_and_shape_c_api.h @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/error_code.h" +#ifdef __cplusplus +extern "C" { +#endif +struct ONNXRuntimeTensorTypeAndShapeInfo; + +//copied from TensorProto::DataType +//Currently, ONNXRuntime doesn't support complex64, complex128, bfloat16 types +typedef enum OnnxRuntimeTensorElementDataType { + ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED = 0, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // maps to c type float + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // maps to c type uint8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // maps to c type int8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // maps to c type uint16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // maps to c type int16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // maps to c type int32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // maps to c type int64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // maps to c++ type std::string + ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, // maps to c type double + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, // maps to c type uint32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, // maps to c type uint64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision +} OnnxRuntimeTensorElementDataType; + +//sync with onnx TypeProto oneof +typedef enum ONNXRuntimeType { + ONNXRUNTIME_TYPE_UNKNOWN, + ONNXRUNTIME_TYPE_TENSOR, + ONNXRUNTIME_TYPE_SEQUENCE, + ONNXRUNTIME_TYPE_MAP, + ONNXRUNTIME_TYPE_OPAQUE, + ONNXRUNTIME_TYPE_SPARSETENSOR, +} ONNXRuntimeType; + +struct ONNXRuntimeTypeInfo; + +/** + * Don't free the returned value + */ +ONNXRUNTIME_API(const struct ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCastTypeInfoToTensorInfo, _In_ struct ONNXRuntimeTypeInfo*); + +/** + * The retured value should be released by calling ONNXRuntimeReleaseObject + */ +ONNXRUNTIME_API(struct ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCreateTensorTypeAndShapeInfo); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeSetTensorElementType, _In_ struct ONNXRuntimeTensorTypeAndShapeInfo*, enum OnnxRuntimeTensorElementDataType type); + +/** + * \param info Created from ONNXRuntimeCreateTensorTypeAndShapeInfo() function + * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. + * \param dim_count length of dim_values + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeSetDims, struct ONNXRuntimeTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); + +ONNXRUNTIME_API(enum OnnxRuntimeTensorElementDataType, ONNXRuntimeGetTensorElementType, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo*); +ONNXRUNTIME_API(size_t, ONNXRuntimeGetNumOfDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info); +ONNXRUNTIME_API(void, ONNXRuntimeGetDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); + +/** + * How many elements does this tensor have. + * May return a negative value + * e.g. + * [] -> 1 + * [1,3,4] -> 12 + * [2,0,4] -> 0 + * [-1,3,4] -> -1 + * return a negative value if unknown. (That this shape contains a symbolic variable which + * represents an unknown dimension.) + */ +ONNXRUNTIME_API(int64_t, ONNXRuntimeGetTensorShapeElementCount, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info); +struct ONNXValue; + +/** + * \param out Should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTensorShapeAndType, _In_ const struct ONNXValue* value, + _Out_ struct ONNXRuntimeTensorTypeAndShapeInfo** out); + +/** + * Get the type information of an ONNXValue + * \param value + * \param out The returned value should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTypeInfo, _In_ const struct ONNXValue* value, struct ONNXRuntimeTypeInfo** out); + +ONNXRUNTIME_API(enum ONNXRuntimeType, ONNXRuntimeGetValueType, _In_ const struct ONNXValue* value); + +#ifdef __cplusplus +} +#endif diff --git a/include/onnxruntime/core/graph/graph_base.h b/include/onnxruntime/core/graph/graph_base.h index 00d9985f5d998..9dc66067fc637 100644 --- a/include/onnxruntime/core/graph/graph_base.h +++ b/include/onnxruntime/core/graph/graph_base.h @@ -206,6 +206,16 @@ class Node { // Get node attributes. const NodeAttributes& GetAttributes() const noexcept; + // get the Graph instance that is instantiated from a GraphProto attribute + // when the main Graph is resolved. returns nullptr if the Graph instance + // has not been instantiated. + const Graph* GetGraphAttribute(const std::string& attr_name) const; + + // get the Graph instance that is instantiated from a GraphProto attribute + // when the main Graph is resolved. returns nullptr if the Graph instance + // has not been instantiated. + Graph* GetMutableGraphAttribute(const std::string& attr_name); + // Indicates on which we will run this node in runtime. // Executor will decide which device that this node will run against // and set it properly. @@ -305,6 +315,9 @@ class Node { const NodeAttributes* attributes, const std::string& domain); + // create a Graph instance for an attribute that contains a GraphProto + void CreateSubgraph(const std::string& attr_name); + // internal only method to allow selected classes to directly alter // the input/output definitions and arg counts Definitions& MutableDefinitions() noexcept; @@ -313,6 +326,8 @@ class Node { // the links between nodes. Relationships& MutableRelationships() noexcept; + const std::vector>& MutableSubgraphs() noexcept { return subgraphs_; } + const Definitions& GetDefinitions() const noexcept { return definitions_; } const Relationships& GetRelationships() const noexcept { return relationships_; } @@ -355,7 +370,11 @@ class Node { // This allows attribute adding and removing. NodeAttributes attributes_; + // Graph that contains this Node Graph* graph_; + + std::unordered_map attr_to_subgraph_map_; + std::vector> subgraphs_; }; #ifdef _MSC_VER @@ -537,13 +556,6 @@ class Graph { Status InlineFunction(Node& node); - // Get the Graph instance for a node that contains a GraphProto attribute in attribute_name. - // Non-const as the Graph instance returned for the subgraph is mutable and owned by this Graph instance. - Graph* GetMutableSubgraph(const NodeIndex index, const std::string& attribute_name); - - // Const version for the above - const Graph* GetSubgraph(const NodeIndex index, const std::string& attribute_name) const; - // when creating a subgraph, record that a NodeArg will come from the outer scope. // This prevents it from being added to the graph inputs. void AddOuterScopeNodeArg(const std::string& name) { @@ -562,6 +574,9 @@ class Graph { graph_output_order_ = outputs; } + // Construct a Graph instance for a subgraph. Inherits some properties from the parent graph. + Graph(Graph& parent_graph, ONNX_NAMESPACE::GraphProto& subgraph_proto); + virtual ~Graph(); private: @@ -581,9 +596,6 @@ class Graph { IOnnxRuntimeOpSchemaCollectionPtr schema_registry, const std::unordered_map& model_functions = {}); - // Construct a Graph instance for a subgraph. Inherits some properties from the parent graph. - Graph(Graph& parent_graph, ONNX_NAMESPACE::GraphProto& subgraph_proto); - // internal use only Graph(ONNX_NAMESPACE::GraphProto* graph_proto, const std::unordered_map& domain_to_version, @@ -621,14 +633,14 @@ class Graph { std::unordered_set inputs_and_initializers; std::unordered_set outer_scope_node_args; std::unordered_map node_name_to_index; - std::unordered_map> node_to_subgraphs_map; + std::unordered_set nodes_with_subgraphs; void Clear() { output_args.clear(); inputs_and_initializers.clear(); outer_scope_node_args.clear(); node_name_to_index.clear(); - node_to_subgraphs_map.clear(); + nodes_with_subgraphs.clear(); } private: @@ -668,10 +680,11 @@ class Graph { common::Status Resolve(bool no_proto_sync_required); - common::Status CreateSubgraphs(); + // Recursively find all subgraphs including nested subgraphs + void FindAllSubgraphs(std::vector& subgraphs); // Iterate this Graph instance and all subgraphs, calling the provided function for each. - common::Status ForThisAndAllSubgraphs(std::function func); + common::Status ForThisAndAllSubgraphs(const std::vector& subgraphs, std::function func); common::Status InferAndVerifyTypeMatch(Node& node, const ONNX_NAMESPACE::OpSchema& op); @@ -787,14 +800,6 @@ class Graph { // the parent graph if this is a subgraph. Graph* parent_graph_; - // entry for node containing subgraph, with value containing attribute_name:Graph pair - // as a node may contain multiple subgraphs (e.g. 'If' has one for both the 'then' and 'else' branches). - using AttributeGraphMap = std::unordered_map; - using SubgraphMap = std::unordered_map; - - SubgraphMap subgraph_map_; - std::vector> subgraphs_; - // NodeArgs that come from outer scope. Used when building a graph so that // these don't get recorded as graph inputs in the GraphProto. std::unordered_set outer_scope_node_arg_names_; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index a6aa9391a59ae..1f449db4e4313 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -11,7 +11,7 @@ #include "core/framework/error_code.h" #include "core/framework/onnx_object.h" #include "core/framework/run_options_c_api.h" -#include "core/session/tensor_type_and_shape_c_api.h" +#include "core/framework/tensor_type_and_shape_c_api.h" #include "allocator.h" #include "session_options_c_api.h" @@ -21,30 +21,10 @@ extern "C" { //Any pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. -typedef enum ONNXRuntimeType { - ONNXRUNTIME_TYPE_TENSOR, - ONNXRUNTIME_TYPE_SEQUENCE, - ONNXRUNTIME_TYPE_MAP, - ONNXRUNTIME_TYPE_OPAQUE, - ONNXRUNTIME_TYPE_ELEMENT, //basic types like float/int32 -} ONNXRuntimeType; - -typedef struct ONNXOpaqueTypeInfo { - char* domain; - char* name; -} ONNXOpaqueTypeInfo; - -//Each ONNX value is a n-ary tree. -//Data is only stored in leaf nodes. -//Every non-leaf node contains a field of ONNXRuntimeType -//Each leaf node is either a tensor, or an ONNXArray. - -/** - * ReleaseONNXEnv function calls ::google::protobuf::ShutdownProtobufLibrary(). - * Therefore, you should only call ReleaseONNXEnv at the end of your program. - * Once you did that, don't call any onnxruntime, onnx or protobuf functions again. - */ -DEFINE_RUNTIME_CLASS(ONNXEnv); +struct ONNXRuntimeEnv; +typedef struct ONNXRuntimeEnv ONNXRuntimeEnv; +//old name +typedef struct ONNXRuntimeEnv* ONNXEnvPtr; typedef enum ONNXRuntimeLoggingLevel { ONNXRUNTIME_LOGGING_LEVEL_kVERBOSE = 0, @@ -58,18 +38,21 @@ typedef void(ONNXRUNTIME_API_STATUSCALL* ONNXRuntimeLoggingFunction)( void* param, ONNXRuntimeLoggingLevel severity, const char* category, const char* logid, const char* code_location, const char* message); /** - * ONNXEnv is process-wise. For each process, only one ONNXEnv can be created. Don't do it multiple times + * ONNXRuntimeEnv is process-wise. For each process, only one ONNXRuntimeEnv can be created. Don't do it multiple times + * \param out Should be freed by `ONNXRuntimeReleaseObject` after use */ ONNXRUNTIME_API_STATUS(ONNXRuntimeInitialize, ONNXRuntimeLoggingLevel default_warning_level, _In_ const char* logid, - _Out_ ONNXEnv** out) + _Out_ ONNXRuntimeEnv** out) ONNXRUNTIME_ALL_ARGS_NONNULL; + /** - * ONNXEnv is process-wise. For each process, only one ONNXEnv can be created. Don't do it multiple times + * ONNXRuntimeEnv is process-wise. For each process, only one ONNXRuntimeEnv can be created. Don't do it multiple times + * \param out Should be freed by `ONNXRuntimeReleaseObject` after use */ ONNXRUNTIME_API_STATUS(ONNXRuntimeInitializeWithCustomLogger, ONNXRuntimeLoggingFunction logging_function, - void* logger_param, ONNXRuntimeLoggingLevel default_warning_level, + _In_opt_ void* logger_param, ONNXRuntimeLoggingLevel default_warning_level, _In_ const char* logid, - _Out_ ONNXEnv** out); + _Out_ ONNXRuntimeEnv** out); DEFINE_RUNTIME_CLASS(ONNXSession); @@ -80,11 +63,11 @@ DEFINE_RUNTIME_CLASS(ONNXSession); // What sort of access is needed to model_path : read or read/write? //TODO: allow loading from an in-memory byte-array #ifdef _WIN32 -ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const wchar_t* model_path, - _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out); +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateInferenceSession, _In_ ONNXRuntimeEnv* env, _In_ const wchar_t* model_path, + _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSession** out); #else -ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const char* model_path, - _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out); +ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateInferenceSession, _In_ ONNXRuntimeEnv* env, _In_ const char* model_path, + _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSession** out); #endif DEFINE_RUNTIME_CLASS(ONNXValue); @@ -93,110 +76,86 @@ DEFINE_RUNTIME_CLASS(ONNXValue); ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateDefaultAllocator, _Out_ ONNXRuntimeAllocator** out); /** - * This function is only for advanced users. In most cases, please use ONNXRuntimeCreateTensorWithDataAsONNXValue - * The returned ONNXValuePtr will keep a reference to allocator, without reference counting + * Create a tensor from an allocator. ReleaseONNXValue will also release the buffer inside the output value + * \param out will keep a reference to the allocator, without reference counting(will be fixed). Should be freed by + * calling ReleaseONNXValue * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx */ ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateTensorAsONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, _In_ const size_t* shape, size_t shape_len, OnnxRuntimeTensorElementDataType type, - _Out_ ONNXValuePtr* out); + _Out_ ONNXValue** out); /** - * p_data is owned by caller. ReleaseTensor won't release p_data. + * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. + * p_data is owned by caller. ReleaseONNXValue won't release p_data. + * \param out Should be freed by calling ReleaseONNXValue */ ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateTensorWithDataAsONNXValue, _In_ const ONNXRuntimeAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, - OnnxRuntimeTensorElementDataType type, _Out_ ONNXValuePtr* out); + OnnxRuntimeTensorElementDataType type, _Out_ ONNXValue** out); /// This function doesn't work with string tensor /// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. -ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTensorMutableData, _In_ ONNXValuePtr value, _Out_ void** out); +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTensorMutableData, _Inout_ ONNXValue* value, _Out_ void** out); /** + * Test if an ONNXValue is a tensor * \return zero, false. non-zero true */ -ONNXRUNTIME_API(int, ONNXRuntimeIsTensor, _In_ ONNXValuePtr value); +ONNXRUNTIME_API(int, ONNXRuntimeIsTensor, _In_ const ONNXValue* value); /** * \param value A tensor created from ONNXRuntimeCreateTensor*** function. * \param s each A string array. Each string in this array must be null terminated. * \param s_len length of s */ -ONNXRUNTIME_API_STATUS(ONNXRuntimeFillStringTensor, _In_ ONNXValuePtr value, _In_ const char* s[], size_t s_len); +ONNXRUNTIME_API_STATUS(ONNXRuntimeFillStringTensor, _In_ ONNXValue* value, _In_ const char* const* s, size_t s_len); /** * \param value A tensor created from ONNXRuntimeCreateTensor*** function. * \param len total data length, not including the trailing '\0' chars. */ -ONNXRUNTIME_API_STATUS(ONNXRuntimeGetStringTensorDataLength, _In_ ONNXValuePtr value, _Out_ size_t* len); +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetStringTensorDataLength, _In_ const ONNXValue* value, _Out_ size_t* len); /** * \param s string contents. Each string is NOT null-terminated. * \param value A tensor created from ONNXRuntimeCreateTensor*** function. * \param s_len total data length, get it from ONNXRuntimeGetStringTensorDataLength */ -ONNXRUNTIME_API_STATUS(ONNXRuntimeGetStringTensorContent, _In_ ONNXValuePtr value, _Out_ void* s, size_t s_len, +ONNXRUNTIME_API_STATUS(ONNXRuntimeGetStringTensorContent, _In_ const ONNXValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len); -/** - * \param out Should be freed by ONNXRuntimeReleaseObject after use - */ -ONNXRUNTIME_API_STATUS(ONNXRuntimeGetTensorShapeAndType, _In_ const ONNXValuePtr, - _Out_ struct ONNXRuntimeTensorTypeAndShapeInfo** out); +DEFINE_RUNTIME_CLASS(ONNXValueList); -//not implemented -//ONNX_RUNTIME_EXPORT int GetPONNXValueDataType(_In_ ONNXValuePtr) NO_EXCEPTION; +ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInference, _Inout_ ONNXSession* sess, + _In_ ONNXRuntimeRunOptions* run_options, + _In_ const char* const* input_names, _In_ const ONNXValue* const* input, size_t input_len, + _In_ const char* const* output_names, size_t output_names_len, _Out_ ONNXValue** output); -DEFINE_RUNTIME_CLASS(ONNXValueList); +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputCount, _In_ const ONNXSession* sess, _Out_ size_t* out); +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputCount, _In_ const ONNXSession* sess, _Out_ size_t* out); -//For InferenceSession run calls, all the input values shouldn't created by allocator -//User should manage the buffer by himself, not allocator +/** + * \param out should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct ONNXRuntimeTypeInfo** out); /** - * \param sess created by ONNXRuntimeCreateInferenceSession function - * \param output must be freed by ReleaseONNXValueListPtr function + * \param out should be freed by ONNXRuntimeReleaseObject after use */ -ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInferenceAndFetchAll, _In_ ONNXSessionPtr sess, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len); -ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInferenceAndFetchAllWithRunOptions, _In_ ONNXSessionPtr sess, - _In_ ONNXRuntimeRunOptionsPtr run_options, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len); -ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInference, _In_ ONNXSessionPtr sess, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _In_ const char* output_names[], size_t output_names_len, _Out_ ONNXValuePtr* output); -ONNXRUNTIME_API_STATUS(ONNXRuntimeRunInferenceWithRunOptions, _In_ ONNXSessionPtr sess, - _In_ ONNXRuntimeRunOptionsPtr run_options, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _In_ const char* output_names[], size_t output_names_len, _Out_ ONNXValuePtr* output); - -ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out); -ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out); - -ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputName, _In_ ONNXSessionPtr sess, size_t index, +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct ONNXRuntimeTypeInfo** out); + +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputName, _In_ const ONNXSession* sess, size_t index, _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** value); -ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputName, _In_ ONNXSessionPtr sess, size_t index, +ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputName, _In_ const ONNXSession* sess, size_t index, _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** value); -//Tree for PONNXType: -//ONNXRUNTIME_TYPE_TENSOR -> ONNXTensorTypeInfo -//ONNXRUNTIME_TYPE_SEQUENCE -> nullptr -//ONNXRUNTIME_TYPE_MAP -> nullptr -//ONNXRUNTIME_TYPE_OPAQUE-> ONNXOpaqueTypeInfo -//ONNXRUNTIME_TYPE_ELEMENT -> nullptr - -//The output value must be freed by ONNXRuntimeNodeDestoryTree -//ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetInputType, _In_ ONNXSessionPtr sess, _Out_ PONNXType* out); -//ONNXRUNTIME_API_STATUS(ONNXRuntimeInferenceSessionGetOutputType, _In_ ONNXSessionPtr sess, _Out_ PONNXType* out); +ONNXRUNTIME_API_STATUS(ONNXRuntimeTensorProtoToONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, + _In_ const void* input, int input_len, _Out_ ONNXValue** out); /** - * Get the n-th value from the List - * \param index starts from zero + * Deprecated. Please use ONNXRuntimeReleaseObject */ -ONNXRUNTIME_API(ONNXValuePtr, ONNXRuntimeONNXValueListGetNthValue, _In_ ONNXValueListPtr list, size_t index); - -ONNXRUNTIME_API_STATUS(ONNXRuntimeTensorProtoToONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, - _In_ const void* input, int input_len, _Out_ ONNXValuePtr* out); +ONNXRUNTIME_API(void, ReleaseONNXEnv, ONNXRuntimeEnv* env); #ifdef __cplusplus } diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index e740d300a8d45..ecd53021e7b4d 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -24,6 +24,26 @@ return ONNXRuntime##NAME(value.get()); \ } +#define DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(TYPE_NAME) \ + namespace std { \ + template <> \ + struct default_delete { \ + void operator()(ONNXRuntime##TYPE_NAME* ptr) { \ + (*reinterpret_cast(ptr))->Release(ptr); \ + } \ + }; \ + } + +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(Env); +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(TypeInfo); +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(Allocator); +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(TensorTypeAndShapeInfo); +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(RunOptions); +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(SessionOptions); +DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT(ProviderFactoryPtr); + +#undef DECLARE_DEFAULT_DELETER_FOR_ONNX_OBJECT + namespace onnxruntime { class SessionOptionsWrapper { private: @@ -32,6 +52,7 @@ class SessionOptionsWrapper { SessionOptionsWrapper(_In_ ONNXEnvPtr env, ONNXRuntimeSessionOptions* p) : value(p, ONNXRuntimeReleaseObject), env_(env){}; public: + //TODO: for the input arg, should we call addref here? SessionOptionsWrapper(_In_ ONNXEnvPtr env) : value(ONNXRuntimeCreateSessionOptions(), ONNXRuntimeReleaseObject), env_(env){}; ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(EnableSequentialExecution) ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL(DisableSequentialExecution) @@ -68,14 +89,14 @@ class SessionOptionsWrapper { return SessionOptionsWrapper(env_, p); } #ifdef _WIN32 - ONNXSessionPtr ONNXRuntimeCreateInferenceSession(_In_ const wchar_t* model_path) { - ONNXSessionPtr ret; + ONNXSession* ONNXRuntimeCreateInferenceSession(_In_ const wchar_t* model_path) { + ONNXSession* ret; ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateInferenceSession(env_, model_path, value.get(), &ret)); return ret; } #else - ONNXSessionPtr ONNXRuntimeCreateInferenceSession(_In_ const char* model_path) { - ONNXSessionPtr ret; + ONNXSession* ONNXRuntimeCreateInferenceSession(_In_ const char* model_path) { + ONNXSession* ret; ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateInferenceSession(env_, model_path, value.get(), &ret)); return ret; } @@ -84,35 +105,25 @@ class SessionOptionsWrapper { ONNXRuntimeAddCustomOp(value.get(), custom_op_path); } }; -inline ONNXValuePtr ONNXRuntimeCreateTensorAsONNXValue(_Inout_ ONNXRuntimeAllocator* env, const std::vector& shape, OnnxRuntimeTensorElementDataType type) { - ONNXValuePtr ret; +inline ONNXValue* ONNXRuntimeCreateTensorAsONNXValue(_Inout_ ONNXRuntimeAllocator* env, const std::vector& shape, OnnxRuntimeTensorElementDataType type) { + ONNXValue* ret; ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateTensorAsONNXValue(env, shape.data(), shape.size(), type, &ret)); return ret; } -inline ONNXValuePtr ONNXRuntimeCreateTensorWithDataAsONNXValue(_In_ const ONNXRuntimeAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, OnnxRuntimeTensorElementDataType type) { - ONNXValuePtr ret; +inline ONNXValue* ONNXRuntimeCreateTensorWithDataAsONNXValue(_In_ const ONNXRuntimeAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, OnnxRuntimeTensorElementDataType type) { + ONNXValue* ret; ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateTensorWithDataAsONNXValue(info, p_data, p_data_len, shape.data(), shape.size(), type, &ret)); return ret; } +inline std::vector GetTensorShape(const ONNXRuntimeTensorTypeAndShapeInfo* info) { + size_t dims = ONNXRuntimeGetNumOfDimensions(info); + std::vector ret(dims); + ONNXRuntimeGetDimensions(info, ret.data(), ret.size()); + return ret; +} } // namespace onnxruntime -#define DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(TYPE_NAME) \ - namespace std { \ - template <> \ - struct default_delete { \ - void operator()(ONNXRuntime##TYPE_NAME* ptr) { \ - (*reinterpret_cast(ptr))->Release(ptr); \ - } \ - }; \ - } - -DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(Allocator); -DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(TensorTypeAndShapeInfo); -DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(RunOptions); -DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(SessionOptions); -DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT(ProviderFactoryPtr); -#undef DECLEAR_DEFAULT_DELETER_FOR_ONNX_OBJECT #undef ONNXRUNTIME_REDIRECT_SIMPLE_FUNCTION_CALL diff --git a/include/onnxruntime/core/session/tensor_type_and_shape_c_api.h b/include/onnxruntime/core/session/tensor_type_and_shape_c_api.h deleted file mode 100644 index 1e010f458e7cd..0000000000000 --- a/include/onnxruntime/core/session/tensor_type_and_shape_c_api.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/framework/error_code.h" -#ifdef __cplusplus -extern "C" { -#endif -struct ONNXRuntimeTensorTypeAndShapeInfo; - -//copied from TensorProto::DataType -typedef enum OnnxRuntimeTensorElementDataType { - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // float - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // uint8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // int8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // uint16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // int16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // int32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // int64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // string - ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // bool - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision - ONNX_TENSOR_ELEMENT_DATA_TYPE_MAX = 17 -} OnnxRuntimeTensorElementDataType; - -/** - * The retured value should be released by calling ONNXRuntimeReleaseObject - */ -ONNXRUNTIME_API(struct ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCreateTensorTypeAndShapeInfo); - -ONNXRUNTIME_API_STATUS(ONNXRuntimeSetTensorElementType, _In_ struct ONNXRuntimeTensorTypeAndShapeInfo*, enum OnnxRuntimeTensorElementDataType type); - -/** - * \param info Created from ONNXRuntimeCreateTensorTypeAndShapeInfo() function - * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. - * \param dim_count length of dim_values - */ -ONNXRUNTIME_API_STATUS(ONNXRuntimeSetDims, struct ONNXRuntimeTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - -ONNXRUNTIME_API(enum OnnxRuntimeTensorElementDataType, ONNXRuntimeGetTensorElementType, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo*); -ONNXRUNTIME_API(size_t, ONNXRuntimeGetNumOfDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info); -ONNXRUNTIME_API(void, ONNXRuntimeGetDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); - -/** - * How many elements does this tensor have. - * May return a negative value - * e.g. - * [] -> 1 - * [1,3,4] -> 12 - * [2,0,4] -> 0 - * [-1,3,4] -> -1 - */ -ONNXRUNTIME_API(int64_t, ONNXRuntimeGetTensorShapeElementCount, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info); -#ifdef __cplusplus -} -#endif diff --git a/onnxruntime/contrib_ops/contrib_ops.cc b/onnxruntime/contrib_ops/contrib_ops.cc index 7d0cb9cba3229..49c8a133eabc7 100644 --- a/onnxruntime/contrib_ops/contrib_ops.cc +++ b/onnxruntime/contrib_ops/contrib_ops.cc @@ -12,8 +12,8 @@ namespace onnxruntime { namespace contrib { using ::ONNX_NAMESPACE::AttributeProto; -using ::ONNX_NAMESPACE::OpSchema; using ::ONNX_NAMESPACE::OPTIONAL; +using ::ONNX_NAMESPACE::OpSchema; void RegisterContribSchemas() { ONNX_CONTRIB_OPERATOR_SCHEMA(SampleOp) @@ -61,12 +61,58 @@ Sample echo operator.)DOC"); "Constrain outputs to boolean tensor") .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput) .SetDoc(R"DOC(Returns which elements of the input are NaN.)DOC"); + + // Operators for linear 8 bit quanitzation support. + ONNX_CONTRIB_OPERATOR_SCHEMA(QuantizeLinear) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Attr("axis", "The axis along which same quantization parameters are applied. It's optional. If it's not specified, it means per-tensor quantization and input 'x_scale' and 'x_zero_point' must be scalars. If it's specified, it means per 'axis' quantization and input 'x_scale' and 'x_zero_point' must be 1-D tensors.", AttributeProto::INT, false) + .Input(0, "x", "N-D full precision Input tensor to be quantized.", "T1") + .Input(1, "y_scale", "Scale for doing quantization to get 'y'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-axis quantization. If it's a 1-D tensor, its number of elements should be equal to the dimension value of 'axis' dimension of input 'x'.", "T1") + .Input(2, "y_zero_point", "Zero point for doing quantization to get 'y'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-axis quantization. If it's a 1-D tensor, its number of elements should be equal to the dimension value of 'axis' dimension of input 'x'.", "T2") + .Output(0, "y", "N-D quantized output tensor. It has same shape as input 'x'.", "T2") + .TypeConstraint( + "T1", + {"tensor(float)"}, + "Constrain 'x', 'y_scale' to float tensors.") + .TypeConstraint( + "T2", + {"tensor(int8)", "tensor(uint8)"}, + "Constrain 'y_zero_point' and 'y' to 8-bit integer tensors.") + .SetDoc(R"DOC( +The linear quantization operator. It consumes a full precision data, a scale, a zero point and computes the quantized data. +The quantization formula is y = (x / y_scale) + y_zero_point. For (x / y_scale), it computes the nearest integer value to arg (in floating-point format), + rounding halfway cases away from zero. Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per 'axis').)DOC"); + + ONNX_CONTRIB_OPERATOR_SCHEMA(DequantizeLinear) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Attr("axis", "the axis along which same quantization parameters are applied. It's optional. If it's not specified, it means per-tensor quantization and input 'x_scale' and 'x_zero_point' must be scalars. If it's specified, it means per 'axis' quantization and input 'x_scale' and 'x_zero_point' must be 1-D tensors.", AttributeProto::INT, false) + .Input(0, "x", "N-D quantized Input tensor to be de-quantized.", "T2") + .Input(1, "x_scale", "Scale for input 'x'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-axis quantization. If it's a 1-D tensor, its number of elements should be equal to the dimension value of 'axis' dimension of input 'x'.", "T1") + .Input(2, "x_zero_point", "Zero point for input 'x'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-axis quantization. If it's a 1-D tensor, its number of elements should be equal to the dimension value of 'axis' dimension of input 'x'.", "T2") + .Output(0, "y", "N-D full precision output tensor. It has same shape as input 'x'.", "T1") + .TypeConstraint( + "T1", + {"tensor(float)"}, + "Constrain 'y', 'x_scale' to float tensors.") + .TypeConstraint( + "T2", + {"tensor(int8)", "tensor(uint8)"}, + "Constrain 'x_zero_point' and 'x' to 8-bit integer tensors.") + .SetDoc(R"DOC( +The linear de-quantization operator. It consumes a quantized data, a scale, a zero point and computes the full precision data. +The dequantization formula is y = (x - x_zero_point) * x_scale. + Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per 'axis').)DOC"); } class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, SampleOp); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, ExpandDims); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, AttnLSTM); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, IsNaN); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DequantizeLinear); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DequantizeLinear); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QuantizeLinear); void RegisterContribKernels(std::function fn) { fn(BuildKernel()); @@ -76,6 +122,9 @@ void RegisterContribKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); } } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/quantize_linear.cc b/onnxruntime/contrib_ops/cpu/quantize_linear.cc new file mode 100644 index 0000000000000..48b90d847b02b --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/quantize_linear.cc @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "contrib_ops/cpu/quantize_linear.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "core/providers/cpu/tensor/cast_op.h" + +namespace onnxruntime { +namespace contrib { + +ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( + DequantizeLinear, + 1, + uint8_t, + KernelDefBuilder() + .TypeConstraint("axis", DataTypeImpl::GetType()) + .TypeConstraint("x", DataTypeImpl::GetTensorType()) + .TypeConstraint("x_scale", DataTypeImpl::GetTensorType()) + .TypeConstraint("x_zero_point", DataTypeImpl::GetTensorType()) + .TypeConstraint("y", DataTypeImpl::GetTensorType()), + DequantizeLinear); + +ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( + DequantizeLinear, + 1, + int8_t, + KernelDefBuilder() + .TypeConstraint("axis", DataTypeImpl::GetType()) + .TypeConstraint("x", DataTypeImpl::GetTensorType()) + .TypeConstraint("x_scale", DataTypeImpl::GetTensorType()) + .TypeConstraint("x_zero_point", DataTypeImpl::GetTensorType()) + .TypeConstraint("y", DataTypeImpl::GetTensorType()), + DequantizeLinear); + +template +// formula is Y = (X - ZeroPoint) * Scale +Status DequantizeLinear::Compute(OpKernelContext* ctx) const { + auto& x = *ctx->Input(0); + auto& x_scale = *ctx->Input(1); + auto& x_zero_point = *ctx->Input(2); + auto& y = *ctx->Output(0, x.Shape()); + + TensorShape shape(0, 0); + std::unique_ptr reshaped_zero_point; + std::unique_ptr reshaped_scale; + + // if an axis was provided, build the shape necessary for broadcasting across that axis + if (has_axis_) { + ONNXRUNTIME_ENFORCE(axis_ < static_cast(x.Shape().NumDimensions()), "axis greater than input data dimension!"); + std::vector shape_; + shape_.push_back(x_zero_point.Size()); + if (axis_ > 0) { + for (int64_t i = axis_ - 1; i >= 0; i--) { + shape_.push_back(1); + } + } + shape = TensorShape(shape_); + + // reshape copies of the inputs for broadcasting. + TensorAllocator tensorAllocatorUint8(*ctx); + reshaped_zero_point = tensorAllocatorUint8.Allocate(shape); + memcpy(reshaped_zero_point->MutableDataRaw(), x_zero_point.DataRaw(), sizeof(T) * x_zero_point.Size()); + + TensorAllocator tensorAllocatorFloat(*ctx); + reshaped_scale = tensorAllocatorFloat.Allocate(shape); + memcpy(reshaped_scale->MutableDataRaw(), x_scale.DataRaw(), sizeof(float) * x_scale.Size()); + } + + TBroadcaster bc(x, has_axis_ ? *reshaped_zero_point : x_zero_point); + TBroadcastOutput output(bc.GetSpanSize(), y); + BroadcastLoop(bc, output, + [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) { + output = (int32_t(input0) - input1.template cast().array()).template cast(); + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) { + output = (input0.template cast().array() - int32_t(input1)).template cast(); + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { + output = (input0.template cast() - input1.template cast()).template cast(); + }); + + TBroadcaster bc2(y, has_axis_ ? *reshaped_scale : x_scale); + TBroadcastOutput output2(bc2.GetSpanSize(), y); + BroadcastLoop(bc2, output2, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = input0 * input1.array(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = input0.array() * input1; }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = input0.array() * input1.array(); }); + + return Status::OK(); +} + +ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( + QuantizeLinear, + 1, + float, + KernelDefBuilder() + .TypeConstraint("axis", DataTypeImpl::GetType()) + .TypeConstraint("x", DataTypeImpl::GetTensorType()) + .TypeConstraint("y_scale", DataTypeImpl::GetTensorType()) + .TypeConstraint("y_zero_point", DataTypeImpl::GetTensorType()) + .TypeConstraint("y", DataTypeImpl::GetTensorType()), + QuantizeLinear); + +// clamp doesn't exist in the version of that we're using, so +// make a local one. +static float clamp(float v, float lo, float hi) { + if (v < lo) return lo; + if (v > hi) return hi; + return v; +} + +template <> +// formula is Y = X / Scale + ZeroPoint +Status QuantizeLinear::Compute(OpKernelContext* ctx) const { + auto& x = *ctx->Input(0); + auto& y_scale = *ctx->Input(1); + auto& y_zero_point = *ctx->Input(2); + auto& y = *ctx->Output(0, x.Shape()); + + TensorShape shape(0, 0); + std::unique_ptr reshaped_scale; + + TensorAllocator tensorAllocator(*ctx); + + // if an axis was provided, build the shape necessary for broadcasting across that axis + if (has_axis_) { + ONNXRUNTIME_ENFORCE(axis_ < static_cast(x.Shape().NumDimensions()), "axis greater than input data dimension!"); + std::vector shape_; + shape_.push_back(y_zero_point.Size()); + if (axis_ > 0) { + for (int64_t i = axis_ - 1; i >= 0; i--) { + shape_.push_back(1); + } + } + shape = TensorShape(shape_); + + reshaped_scale = tensorAllocator.Allocate(shape); + memcpy(reshaped_scale->MutableDataRaw(), y_scale.DataRaw(), sizeof(float) * y_scale.Size()); + } + + std::unique_ptr W = tensorAllocator.Allocate(x.Shape()); + Tensor* pW = W.get(); + + TBroadcaster bc(x, has_axis_ ? *reshaped_scale : y_scale); + TBroadcastOutput output2(bc.GetSpanSize(), *pW); + BroadcastLoop(bc, output2, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { output = (input0 / input1.array()).round(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = (input0.array() / input1).round(); }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { output = (input0.array() / input1.array()).round(); }); + + std::unique_ptr Zf = tensorAllocator.Allocate(has_axis_ ? shape : y_zero_point.Shape()); + Tensor* pZf = Zf.get(); + CastData(&y_zero_point, pZf, has_axis_ ? shape : y_zero_point.Shape()); + + TBroadcaster bc2(*pW, *pZf); + TBroadcastOutput output(bc2.GetSpanSize(), y); + BroadcastLoop(bc2, output, + [](EigenVectorMap output, float input0, ConstEigenVectorMap input1) { + for (std::ptrdiff_t i = 0; i < output.size(); i++) { + output[i] = uint8_t(clamp(input0 + float(input1[i]), 0.0f, float(UINT8_MAX))); + } + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { + for (std::ptrdiff_t i = 0; i < output.size(); i++) { + output[i] = uint8_t(clamp(input0[i] + float(input1), 0.0f, float(UINT8_MAX))); + } + }, + [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) { + for (std::ptrdiff_t i = 0; i < output.size(); i++) { + output[i] = uint8_t(clamp(input0[i] + float(input1[i]), 0.0f, float(UINT8_MAX))); + } + }); + + return Status::OK(); +} +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/quantize_linear.h b/onnxruntime/contrib_ops/cpu/quantize_linear.h new file mode 100644 index 0000000000000..1175e88fcf045 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/quantize_linear.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math_cpuonly.h" + +namespace onnxruntime { +namespace contrib { + +template +class DequantizeLinear final : public OpKernel { + public: + DequantizeLinear(const OpKernelInfo& info) : OpKernel(info) { + has_axis_ = info.GetAttr("axis", &axis_).IsOK(); + } + + Status Compute(OpKernelContext* context) const override; + + private: + int64_t axis_; + bool has_axis_; +}; + +template +class QuantizeLinear final : public OpKernel { + public: + QuantizeLinear(const OpKernelInfo& info) : OpKernel(info) { + has_axis_ = info.GetAttr("axis", &axis_).IsOK(); + } + + Status Compute(OpKernelContext* context) const override; + + private: + int64_t axis_; + bool has_axis_; +}; +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/core/codegen/tvm/tvm_kernel.h b/onnxruntime/core/codegen/tvm/tvm_kernel.h index 4ce4d3755c8a2..c6d745f532fe7 100644 --- a/onnxruntime/core/codegen/tvm/tvm_kernel.h +++ b/onnxruntime/core/codegen/tvm/tvm_kernel.h @@ -101,8 +101,8 @@ class TVMKernel : public OpKernel { tvm::TVMRetValue rvalue; try { evaluate_func_.CallPacked(tvm_args, &rvalue); - } catch (std::exception ex) { - return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::FAIL, "TVM run failed."); + } catch (std::exception& ex) { + return ONNXRUNTIME_MAKE_STATUS(ONNXRUNTIME, FAIL, "TVM run failed:", ex.what()); } if (rvalue.type_code() != kNull) { return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::FAIL, "TVM return not null"); // TODO: get error code. diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index d033e27e84dc0..d4d170f5993fc 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -98,6 +98,7 @@ std::ostream& operator<<(std::ostream& out, std::pair& outer_scope_node_args, const ExecutionProviders& providers, const KernelRegistryManager& kernel_registry, const MLValueNameIdxMap& mlvalue_name_idx_map, @@ -105,7 +106,8 @@ class PlannerImpl { SequentialExecutionPlan& plan) : context_{context}, plan_{plan}, - graph_viewer_(graph_viewer), + graph_viewer_{graph_viewer}, + outer_scope_node_args_{outer_scope_node_args}, execution_providers_{providers}, kernel_registry_{kernel_registry}, mlvalue_name_idx_map_{mlvalue_name_idx_map} { @@ -118,6 +120,7 @@ class PlannerImpl { SequentialExecutionPlan& plan_; const onnxruntime::GraphViewer& graph_viewer_; + const std::vector& outer_scope_node_args_; const ExecutionProviders& execution_providers_; const KernelRegistryManager& kernel_registry_; @@ -335,6 +338,12 @@ class PlannerImpl { UseCount(index)++; // Models caller's usage post-inference; ensures it will not be reused. } + for (auto node_arg : outer_scope_node_args_) { + MLValueIndex index = Index(node_arg->Name()); + ProcessDef(index, node_arg); + UseCount(index)++; // ensure will not be re-used as this graph does not own the buffer + } + // All initializers should be treated as input for (const auto& pair : graph_viewer_.GetAllInitializedTensors()) { const auto& initializer_name = pair.first; @@ -444,14 +453,23 @@ class PlannerImpl { // Identify allocation/deallocation plan for every ml-value + auto setup_preexisting = [this](const NodeArg* node_arg) { + auto input_index = Index(node_arg->Name()); + SequentialExecutionPlan::AllocPlanPerValue& thisplan = AllocPlan(input_index); + thisplan.alloc_kind = AllocKind::kPreExisting; + thisplan.value_type = utils::GetMLDataType(*node_arg); + }; + // inputs of the graph: // An input ml-value's data is owned by the caller (of InferenceSession::Run()) // It must be allocated by the caller, and will not be reused during inference. for (auto graph_input : graph_viewer_.GetInputs()) { - auto input_index = Index(graph_input->Name()); - SequentialExecutionPlan::AllocPlanPerValue& thisplan = AllocPlan(input_index); - thisplan.alloc_kind = AllocKind::kPreExisting; - thisplan.value_type = utils::GetMLDataType(*graph_input); + setup_preexisting(graph_input); + } + + // outer scope node args are treated the same as graph inputs + for (auto outer_scope_node_arg : outer_scope_node_args_) { + setup_preexisting(outer_scope_node_arg); } GeneratePlanForWeights(); @@ -565,7 +583,7 @@ Status PlannerImpl::CreatePlan() { // Determine execution order: we use the default topological sort order for now. We can later // explore more efficient orderings (from a memory usage perspective). for (auto n : p_graph_nodes) { - plan_.execution_plan.emplace_back(n); + plan_.execution_plan.emplace_back(n); } // compute use counts for all ml-values @@ -581,6 +599,7 @@ Status PlannerImpl::CreatePlan() { } Status SequentialPlanner::CreatePlan(const onnxruntime::GraphViewer& graph_viewer, + const std::vector& outer_scope_node_args, const ExecutionProviders& providers, const KernelRegistryManager& kernel_registry, const MLValueNameIdxMap& mlvalue_name_idx_map, @@ -589,7 +608,8 @@ Status SequentialPlanner::CreatePlan(const onnxruntime::GraphViewer& graph_viewe // allocate/reset here so we know it's clean plan = std::make_unique(); - PlannerImpl planner(graph_viewer, providers, kernel_registry, mlvalue_name_idx_map, context, *plan); + PlannerImpl planner(graph_viewer, outer_scope_node_args, + providers, kernel_registry, mlvalue_name_idx_map, context, *plan); return planner.CreatePlan(); } diff --git a/onnxruntime/core/framework/allocation_planner.h b/onnxruntime/core/framework/allocation_planner.h index d9eab0f5d660a..4e5db48c1e484 100644 --- a/onnxruntime/core/framework/allocation_planner.h +++ b/onnxruntime/core/framework/allocation_planner.h @@ -51,6 +51,7 @@ class SequentialPlanner { public: // This API allows user to provide a custom planner context. static Status CreatePlan(const onnxruntime::GraphViewer& graph, + const std::vector& outer_scope_node_args, const ExecutionProviders& providers, const KernelRegistryManager& kernel_registry, const MLValueNameIdxMap& mlvalue_name_idx_map, @@ -60,12 +61,13 @@ class SequentialPlanner { // This uses a standard planner context and is meant to be the primary API for creating a plan // as the context is primarily used in test scenarios. static Status CreatePlan(const onnxruntime::GraphViewer& graph, + const std::vector& outer_scope_node_args, const ExecutionProviders& providers, const KernelRegistryManager& kernel_registry, const MLValueNameIdxMap& mlvalue_name_idx_map, std::unique_ptr& plan) { SequentialPlannerContext context; - return CreatePlan(graph, providers, kernel_registry, mlvalue_name_idx_map, context, plan); + return CreatePlan(graph, outer_scope_node_args, providers, kernel_registry, mlvalue_name_idx_map, context, plan); } }; diff --git a/onnxruntime/core/framework/cblas.h b/onnxruntime/core/framework/cblas.h index 78b5e2255cdd6..fd3f6823ef874 100644 --- a/onnxruntime/core/framework/cblas.h +++ b/onnxruntime/core/framework/cblas.h @@ -4,12 +4,7 @@ // This is the exact cblas.h header file, placed here purely in order to get // the enums. -//#include "caffe2/core/macros.h" - #ifndef CBLAS_H -#ifdef CAFFE2_USE_MKL -#include -#else // CAFFE2_USE_MKL #ifndef CBLAS_ENUM_DEFINED_H #define CBLAS_ENUM_DEFINED_H @@ -601,6 +596,5 @@ void cblas_zher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, int cblas_errprn(int ierr, int info, char* form, ...); -#endif /* end #ifdef CBLAS_ENUM_ONLY */ -#endif // CAFFE2_USE_MKL +#endif /* end #ifdef CBLAS_ENUM_ONLY */ #endif diff --git a/onnxruntime/core/framework/onnx_object.cc b/onnxruntime/core/framework/onnx_object.cc index c2ae87b97b0a7..67da0a216520c 100644 --- a/onnxruntime/core/framework/onnx_object.cc +++ b/onnxruntime/core/framework/onnx_object.cc @@ -2,11 +2,20 @@ // Licensed under the MIT License. #include "core/framework/onnx_object.h" +#include ONNXRUNTIME_API(uint32_t, ONNXRuntimeAddRefToObject, void* ptr) { return (*static_cast(ptr))->AddRef(ptr); } + ONNXRUNTIME_API(uint32_t, ONNXRuntimeReleaseObject, void* ptr) { if (ptr == nullptr) return 0; return (*static_cast(ptr))->Release(ptr); } + +namespace { +struct ObjectImpl { + const ONNXObject* const cls; + std::atomic_int ref_count; +}; +} // namespace diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.cc b/onnxruntime/core/framework/onnxruntime_typeinfo.cc new file mode 100644 index 0000000000000..f77c379b0618e --- /dev/null +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.cc @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//this file contains implementations of the C API + +#include "onnxruntime_typeinfo.h" +#include +#include "core/framework/tensor.h" +#include "core/graph/onnx_protobuf.h" + +using onnxruntime::DataTypeImpl; +using onnxruntime::MLFloat16; +using onnxruntime::Tensor; +using onnxruntime::TensorShape; + +ONNXRuntimeTypeInfo::ONNXRuntimeTypeInfo(ONNXRuntimeType type1, void* data1) noexcept : type(type1), data(data1) { +} + +ONNXRuntimeTypeInfo::~ONNXRuntimeTypeInfo() { + assert(ref_count == 0); + ONNXRuntimeReleaseObject(data); +} + +ONNXRUNTIME_API(const struct ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCastTypeInfoToTensorInfo, _In_ struct ONNXRuntimeTypeInfo* input) { + return input->type == ONNXRUNTIME_TYPE_TENSOR ? reinterpret_cast(input->data) : nullptr; +} + +ONNXStatusPtr GetTensorShapeAndType(const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, ONNXRuntimeTensorTypeAndShapeInfo** out); + +ONNXStatusPtr ONNXRuntimeTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, ONNXRuntimeTypeInfo** out) { + if (input == nullptr) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_UNKNOWN, nullptr); + return nullptr; + } + if (input == DataTypeImpl::GetType()) { + ONNXRuntimeTensorTypeAndShapeInfo* info = nullptr; + if (tensor_data_type != nullptr) { + ONNXStatusPtr st = GetTensorShapeAndType(shape, tensor_data_type, &info); + if (st != nullptr) return st; + } + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_TENSOR, info); + return nullptr; + } + if (input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType()) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_MAP, nullptr); + return nullptr; + } + if (input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType()) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_SEQUENCE, nullptr); + return nullptr; + } + return CreateONNXStatus(ONNXRUNTIME_NOT_IMPLEMENTED, "not implemented"); +} + +const DataTypeImpl* ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { + switch (type) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_BOOL: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT32: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_STRING: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT8: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT16: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT16: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_INT64: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT32: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_UINT64: + return DataTypeImpl::GetType(); + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: + return DataTypeImpl::GetType(); + default: + ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, ":tensor type ", type, " is not supported"); + } +} + +ONNXStatusPtr ONNXRuntimeTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, ONNXRuntimeTypeInfo** out) { + if (input->has_tensor_type()) { + const ::onnx::TypeProto_Tensor& onnx_tensor_info = input->tensor_type(); + const DataTypeImpl* type = ElementTypeFromProto(onnx_tensor_info.elem_type()); + ONNXStatusPtr st; + ONNXRuntimeTensorTypeAndShapeInfo* info = nullptr; + if (onnx_tensor_info.has_shape()) { + const ::onnx::TensorShapeProto& s = onnx_tensor_info.shape(); + std::vector shape_data(s.dim_size()); + for (int i = 0; i != s.dim_size(); ++i) { + auto& t = s.dim(i); + shape_data[i] = t.has_dim_value() ? t.dim_value() : -1; + } + st = GetTensorShapeAndType(reinterpret_cast(&shape_data), type, &info); + } else { + st = GetTensorShapeAndType(nullptr, type, &info); + } + + if (st != nullptr) return st; + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_TENSOR, info); + return nullptr; + } + if (input->has_sequence_type()) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_SEQUENCE, nullptr); + return nullptr; + } + if (input->has_map_type()) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_MAP, nullptr); + return nullptr; + } + if (input->has_opaque_type()) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_OPAQUE, nullptr); + return nullptr; + } + if (input->has_sparse_tensor_type()) { + *out = new ONNXRuntimeTypeInfo(ONNXRUNTIME_TYPE_SPARSETENSOR, nullptr); + return nullptr; + } + return CreateONNXStatus(ONNXRUNTIME_NOT_IMPLEMENTED, "not implemented"); +} \ No newline at end of file diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.h b/onnxruntime/core/framework/onnxruntime_typeinfo.h new file mode 100644 index 0000000000000..43b7977548eaa --- /dev/null +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.h @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/onnx_object.h" +#include "core/framework/onnx_object_cxx.h" +#include "core/framework/tensor_type_and_shape_c_api.h" +#include + +namespace onnxruntime { +class DataTypeImpl; +class TensorShape; +} // namespace onnxruntime + +namespace onnx { +class TypeProto; +} + +/** + * the equivalent of onnx::TypeProto + * This class is mainly for the C API + */ +struct ONNXRuntimeTypeInfo : public onnxruntime::ObjectBase { + public: + friend class onnxruntime::ObjectBase; + + ONNXRuntimeType type = ONNXRUNTIME_TYPE_UNKNOWN; + //owned by this + void* data = nullptr; + ONNXRuntimeTypeInfo(const ONNXRuntimeTypeInfo& other) = delete; + ONNXRuntimeTypeInfo& operator=(const ONNXRuntimeTypeInfo& other) = delete; + + static ONNXStatusPtr FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const onnxruntime::TensorShape* shape, + const onnxruntime::DataTypeImpl* tensor_data_type, ONNXRuntimeTypeInfo** out); + static ONNXStatusPtr FromDataTypeImpl(const onnx::TypeProto*, ONNXRuntimeTypeInfo** out); + + private: + ONNXRuntimeTypeInfo(ONNXRuntimeType type, void* data) noexcept; + ~ONNXRuntimeTypeInfo(); +}; diff --git a/onnxruntime/core/framework/run_options.cc b/onnxruntime/core/framework/run_options.cc index 812570396bab8..9d1eb4c0640da 100644 --- a/onnxruntime/core/framework/run_options.cc +++ b/onnxruntime/core/framework/run_options.cc @@ -6,26 +6,6 @@ #include #include -uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseRunOptions(void* this_) { - ONNXRuntimeRunOptions* this_ptr = static_cast(this_); - if (--this_ptr->ref_count == 0) - delete this_ptr; - return 0; -} - -uint32_t ONNXRUNTIME_API_STATUSCALL AddRefRunOptions(void* this_) { - ONNXRuntimeRunOptions* this_ptr = static_cast(this_); - ++this_ptr->ref_count; - return 0; -} - -constexpr ONNXObject mkl_cls = { - AddRefRunOptions, - ReleaseRunOptions, -}; - -ONNXRuntimeRunOptions::ONNXRuntimeRunOptions() : cls(&mkl_cls), ref_count(1), terminate(false) { -} ONNXRUNTIME_API(ONNXRuntimeRunOptions*, ONNXRuntimeCreateRunOptions) { std::unique_ptr options = std::make_unique(); diff --git a/onnxruntime/core/framework/session_state_initializer.cc b/onnxruntime/core/framework/session_state_initializer.cc index cbba7164c30dd..5cef936a71514 100644 --- a/onnxruntime/core/framework/session_state_initializer.cc +++ b/onnxruntime/core/framework/session_state_initializer.cc @@ -70,31 +70,46 @@ SessionStateInitializer::SessionStateInitializer(onnxruntime::Graph& graph, common::Status SessionStateInitializer::CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, const InsertCastTransformer& insert_cast_transformer, + const std::vector& outer_scope_node_args, bool enable_sequential_execution) { ONNXRUNTIME_RETURN_IF_ERROR(TransformGraph(graph_, graph_transformation_manager, execution_providers_, kernel_registry_manager_, insert_cast_transformer)); + // After transformation/partitioning, the graph now is fixed and graph viewer is created and set for execution. session_state_.SetGraphViewer(std::make_unique(graph_)); + + auto& mlvalue_name_idx_map = session_state_.GetMLValueNameIdxMap(); + // populate the SessionState MLValueNameIdxMap - ONNXRUNTIME_RETURN_IF_ERROR(SaveMLValueNameIndexMapping(graph_, - session_state_.GetMLValueNameIdxMap(), - logger_)); + ONNXRUNTIME_RETURN_IF_ERROR(SaveMLValueNameIndexMapping(graph_, mlvalue_name_idx_map, logger_)); + + // remove any outer scope args we don't know about. this can happen if a node contains multiple subgraphs. + std::vector valid_outer_scope_node_args; + std::for_each(outer_scope_node_args.cbegin(), outer_scope_node_args.cend(), + [&mlvalue_name_idx_map, &valid_outer_scope_node_args](const NodeArg* node_arg) { + int idx; + if (mlvalue_name_idx_map.GetIdx(node_arg->Name(), idx).IsOK()) { + valid_outer_scope_node_args.push_back(node_arg); + }; + }); std::unique_ptr exec_plan; if (enable_sequential_execution) { // CreatePlan will create a new SequentialExecutionPlan instance that we will // save into the session state. - ONNXRUNTIME_RETURN_IF_ERROR(SequentialPlanner::CreatePlan(graph_, execution_providers_, kernel_registry_manager_, - session_state_.GetMLValueNameIdxMap(), exec_plan)); + ONNXRUNTIME_RETURN_IF_ERROR( + SequentialPlanner::CreatePlan(graph_, valid_outer_scope_node_args, execution_providers_, + kernel_registry_manager_, mlvalue_name_idx_map, exec_plan)); session_state_.SetExecutionPlan(std::move(exec_plan)); } else { // Parallel execution still uses same allocation plan, but has limitation of memory buffer reuse. SequentialPlannerContext context(true /* enable parallel execution */); - ONNXRUNTIME_RETURN_IF_ERROR(SequentialPlanner::CreatePlan(graph_, execution_providers_, kernel_registry_manager_, - session_state_.GetMLValueNameIdxMap(), context, exec_plan)); + ONNXRUNTIME_RETURN_IF_ERROR( + SequentialPlanner::CreatePlan(graph_, valid_outer_scope_node_args, execution_providers_, + kernel_registry_manager_, mlvalue_name_idx_map, context, exec_plan)); session_state_.SetExecutionPlan(std::move(exec_plan)); } diff --git a/onnxruntime/core/framework/session_state_initializer.h b/onnxruntime/core/framework/session_state_initializer.h index 6643ab5812255..c0540e3f379df 100644 --- a/onnxruntime/core/framework/session_state_initializer.h +++ b/onnxruntime/core/framework/session_state_initializer.h @@ -8,15 +8,13 @@ #include "core/framework/tensor.h" namespace onnxruntime { +class ExecutionProviders; class Graph; class GraphTransformerManager; -} // namespace onnxruntime - -namespace onnxruntime { -class SessionState; -class ExecutionProviders; -class KernelRegistryManager; class InsertCastTransformer; +class KernelRegistryManager; +class NodeArg; +class SessionState; namespace logging { class Logger; @@ -33,6 +31,7 @@ class SessionStateInitializer { // First perform any transformations and create the execution plan common::Status CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, const InsertCastTransformer& insert_cast_transformer, + const std::vector& outer_scope_node_args, bool enable_sequential_execution); // initialize tensors, and save. save kernels and input/output node mappings diff --git a/onnxruntime/core/framework/tensor_type_and_shape.cc b/onnxruntime/core/framework/tensor_type_and_shape.cc new file mode 100644 index 0000000000000..0719130245577 --- /dev/null +++ b/onnxruntime/core/framework/tensor_type_and_shape.cc @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensor_type_and_shape_c_api.h" +#include "core/framework/onnx_object.h" +#include "core/framework/tensor_shape.h" +#include "core/framework/ml_value.h" +#include "core/framework/onnxruntime_typeinfo.h" + +#include +#include +#include + +using onnxruntime::DataTypeImpl; +using onnxruntime::MLFloat16; +using onnxruntime::Tensor; + +struct ONNXRuntimeTensorTypeAndShapeInfo : public onnxruntime::ObjectBase { + public: + friend class onnxruntime::ObjectBase; + + OnnxRuntimeTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + onnxruntime::TensorShape shape; + + static ONNXRuntimeTensorTypeAndShapeInfo* Create() { + return new ONNXRuntimeTensorTypeAndShapeInfo(); + } + + ONNXRuntimeTensorTypeAndShapeInfo(const ONNXRuntimeTensorTypeAndShapeInfo& other) = delete; + ONNXRuntimeTensorTypeAndShapeInfo& operator=(const ONNXRuntimeTensorTypeAndShapeInfo& other) = delete; + + private: + ONNXRuntimeTensorTypeAndShapeInfo() = default; + ~ONNXRuntimeTensorTypeAndShapeInfo() { + assert(ref_count == 0); + } +}; + +#define API_IMPL_BEGIN try { +#define API_IMPL_END \ + } \ + catch (std::exception & ex) { \ + return CreateONNXStatus(ONNXRUNTIME_RUNTIME_EXCEPTION, ex.what()); \ + } + +ONNXRUNTIME_API(ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCreateTensorTypeAndShapeInfo) { + return ONNXRuntimeTensorTypeAndShapeInfo::Create(); +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeSetTensorElementType, _In_ ONNXRuntimeTensorTypeAndShapeInfo* this_ptr, enum OnnxRuntimeTensorElementDataType type) { + API_IMPL_BEGIN + this_ptr->type = type; + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeSetDims, _In_ ONNXRuntimeTensorTypeAndShapeInfo* this_ptr, _In_ const int64_t* dim_values, size_t dim_count) { + API_IMPL_BEGIN + this_ptr->shape = onnxruntime::TensorShape(dim_values, dim_count); + return nullptr; + API_IMPL_END +} + +ONNXRUNTIME_API(enum OnnxRuntimeTensorElementDataType, ONNXRuntimeGetTensorElementType, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info) { + return info->type; +} + +ONNXRUNTIME_API(size_t, ONNXRuntimeGetNumOfDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info) { + return info->shape.NumDimensions(); +} + +ONNXRUNTIME_API(void, ONNXRuntimeGetDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) { + info->shape.CopyDims(dim_values, dim_values_length); +} + +ONNXRUNTIME_API(int64_t, ONNXRuntimeGetTensorShapeElementCount, _In_ const ONNXRuntimeTensorTypeAndShapeInfo* this_ptr) { + return this_ptr->shape.Size(); +} + +struct ONNXValue; + +namespace { +inline OnnxRuntimeTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( + const onnxruntime::DataTypeImpl* cpp_type) { + OnnxRuntimeTensorElementDataType type; + if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; + } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; + } else { + type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + } + return type; +} +} // namespace + +ONNXStatusPtr GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, ONNXRuntimeTensorTypeAndShapeInfo** out) { + OnnxRuntimeTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type); + if (ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED == type) { + return CreateONNXStatus(ONNXRUNTIME_FAIL, "Not implemented"); + } + ONNXRuntimeTensorTypeAndShapeInfo* ret = ONNXRuntimeCreateTensorTypeAndShapeInfo(); + auto status = ONNXRuntimeSetTensorElementType(ret, type); + if (status != nullptr) { + ONNXRuntimeReleaseObject(ret); + return status; + } + if (shape != nullptr) { + status = ONNXRuntimeSetDims(ret, shape->GetDims().data(), shape->GetDims().size()); + if (status != nullptr) { + ONNXRuntimeReleaseObject(ret); + return status; + } + } + *out = ret; + return nullptr; +} + +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorShapeAndType, _In_ const ONNXValue* value, + _Out_ ONNXRuntimeTensorTypeAndShapeInfo** out) { + API_IMPL_BEGIN + auto v = reinterpret_cast(value); + const onnxruntime::Tensor& tensor = v->Get(); + return GetTensorShapeAndType(&tensor.Shape(), tensor.DataType(), out); + API_IMPL_END +} + +ONNXRUNTIME_API(enum ONNXRuntimeType, ONNXRuntimeGetValueType, _In_ const ONNXValue* value) { + try { + auto v = reinterpret_cast(value); + onnxruntime::MLDataType type = v->Type(); + ONNXRuntimeTypeInfo* out; + ONNXStatusPtr ptr = ONNXRuntimeTypeInfo::FromDataTypeImpl(type, nullptr, nullptr, &out); + if (ptr != nullptr) { + ReleaseONNXStatus(ptr); + return ONNXRUNTIME_TYPE_UNKNOWN; + } + ONNXRuntimeType ret = out->type; + ONNXRuntimeReleaseObject(out); + return ret; + } catch (std::exception&) { + return ONNXRUNTIME_TYPE_UNKNOWN; + } +} + +/** + * Get the type information of an ONNXValue + * \param value + * \return The returned value should be freed by ONNXRuntimeReleaseObject after use + */ +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTypeInfo, _In_ const ONNXValue* value, struct ONNXRuntimeTypeInfo** out) { + auto v = reinterpret_cast(value); + onnxruntime::MLDataType type = v->Type(); + if (type == nullptr) { + *out = nullptr; + return nullptr; + } + if (type == DataTypeImpl::GetType()) { + const onnxruntime::Tensor& tensor = v->Get(); + const onnxruntime::TensorShape& shape = tensor.Shape(); + return ONNXRuntimeTypeInfo::FromDataTypeImpl(type, &shape, tensor.DataType(), out); + } + return ONNXRuntimeTypeInfo::FromDataTypeImpl(type, nullptr, nullptr, out); +} \ No newline at end of file diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 622f1ee9112fb..363aed31cb675 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -80,22 +80,20 @@ const TypeProto* NodeArg::TypeAsProto() const noexcept { } const TensorShapeProto* NodeArg::Shape() const { - if (!node_arg_info_.has_type()) { - return nullptr; - } - - const auto typeCase = node_arg_info_.type().value_case(); + const TypeProto* type = TypeAsProto(); + if (type == nullptr) return nullptr; + const auto typeCase = type->value_case(); switch (typeCase) { case TypeProto::kTensorType: { - if (node_arg_info_.type().tensor_type().has_shape()) { - return &(node_arg_info_.type().tensor_type().shape()); + if (type->tensor_type().has_shape()) { + return &(type->tensor_type().shape()); } else { return nullptr; } } case TypeProto::kSparseTensorType: { - if (node_arg_info_.type().sparse_tensor_type().has_shape()) { - return &(node_arg_info_.type().sparse_tensor_type().shape()); + if (type->sparse_tensor_type().has_shape()) { + return &(type->sparse_tensor_type().shape()); } else { return nullptr; } @@ -368,6 +366,12 @@ void Node::Init(const std::string& name, if (attributes) { attributes_ = *attributes; + + for (auto& name_to_attr : attributes_) { + if (name_to_attr.second.has_g()) { + CreateSubgraph(name_to_attr.first); + } + } } } @@ -385,6 +389,17 @@ Node::Relationships& Node::MutableRelationships() noexcept { return relationships_; } +void Node::CreateSubgraph(const std::string& attr_name) { + auto attr = attributes_.find(attr_name); + + if (attr != attributes_.cend() && attr->second.has_g()) { + GraphProto& mutable_graph = *attr->second.mutable_g(); + std::unique_ptr subgraph{new Graph(*graph_, mutable_graph)}; + attr_to_subgraph_map_[attr_name] = subgraph.get(); + subgraphs_.push_back(std::move(subgraph)); + } +} + void Node::AddAttribute(const std::string& attr_name, const AttributeProto& value) { graph_->SetGraphResolveNeeded(); graph_->SetGraphProtoSyncNeeded(); @@ -427,11 +442,22 @@ void Node::AddAttribute(const std::string& attr_name, const AttributeProto& valu attributes_[attr_name] = a; \ }; +void Node::AddAttribute(const std::string& attr_name, const GraphProto& value) { + graph_->SetGraphResolveNeeded(); + graph_->SetGraphProtoSyncNeeded(); + AttributeProto a; + a.set_name(attr_name); + a.set_type(AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPH); + *a.mutable_g() = value; + attributes_[attr_name] = a; + + CreateSubgraph(attr_name); +}; + ADD_BASIC_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT, f) ADD_BASIC_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INT, i) ADD_BASIC_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRING, s) ADD_ATTR_IMPL(TensorProto, AttributeProto_AttributeType::AttributeProto_AttributeType_TENSOR, t) -ADD_ATTR_IMPL(GraphProto, AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPH, g) ADD_LIST_ATTR_IMPL(float, AttributeProto_AttributeType::AttributeProto_AttributeType_FLOATS, floats) ADD_LIST_ATTR_IMPL(int64_t, AttributeProto_AttributeType::AttributeProto_AttributeType_INTS, ints) ADD_LIST_ATTR_IMPL(std::string, AttributeProto_AttributeType::AttributeProto_AttributeType_STRINGS, strings) @@ -503,6 +529,21 @@ const NodeAttributes& Node::GetAttributes() const noexcept { return attributes_; } +Graph* Node::GetMutableGraphAttribute(const std::string& attr_name) { + Graph* subgraph = nullptr; + + const auto& entry = attr_to_subgraph_map_.find(attr_name); + if (entry != attr_to_subgraph_map_.cend()) { + subgraph = entry->second; + } + + return subgraph; +} + +const Graph* Node::GetGraphAttribute(const std::string& attr_name) const { + return const_cast(this)->GetMutableGraphAttribute(attr_name); +} + void Node::ForEachDef(std::function func) const { for (const auto* arg : InputDefs()) { if (arg->Exists()) @@ -716,7 +757,7 @@ Status Graph::VerifyNoDuplicateName() { common::Status Graph::SetOuterScopeNodeArgs(const std::unordered_set& outer_scope_node_args) { resolve_context_.outer_scope_node_args = outer_scope_node_args; - if (!resolve_context_.node_to_subgraphs_map.empty()) { + if (!resolve_context_.nodes_with_subgraphs.empty()) { // Build the list of NodeArg's that are valid for a subgraph of this GraphBase instance: // - outer scope for this graph // - any inputs/initializers from this graph @@ -737,8 +778,8 @@ common::Status Graph::SetOuterScopeNodeArgs(const std::unordered_set& entry) { return entry.first; }); - for (auto node_subgraphs : resolve_context_.node_to_subgraphs_map) { - for (auto* subgraph : node_subgraphs.second) { + for (auto* node : resolve_context_.nodes_with_subgraphs) { + for (auto& subgraph : node->MutableSubgraphs()) { auto status = subgraph->SetOuterScopeNodeArgs(node_args_in_scope_for_subgraph); ONNXRUNTIME_RETURN_IF_ERROR(status); } @@ -834,13 +875,14 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c std::unordered_set inner_nodes; // recurse into subgraphs first so we can update any nodes in this graph that are used by those subgraphs - if (!resolve_context_.node_to_subgraphs_map.empty()) { - for (auto nodeid_to_subgraphs : resolve_context_.node_to_subgraphs_map) { - for (auto* subgraph : nodeid_to_subgraphs.second) { + if (!resolve_context_.nodes_with_subgraphs.empty()) { + for (auto* node : resolve_context_.nodes_with_subgraphs) { + for (auto& subgraph : node->MutableSubgraphs()) { std::vector node_args_consumed; subgraph->BuildConnections(node_args_consumed); for (auto& node_arg_name : node_args_consumed) { + bool node_arg_in_parent_graph = false; const auto* node_arg = GetNodeArg(node_arg_name); if (node_arg == nullptr) { @@ -858,22 +900,30 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c node_arg = parent_graph_->GetNodeArgIncludingParentGraphs(node_arg_name); + // make sure the node arg is found in the parent graph/s if (!node_arg) { return ONNXRUNTIME_MAKE_STATUS( ONNXRUNTIME, INVALID_GRAPH, "Failed to find NodeArg in all parent graphs. Name=", node_arg_name, " Graph may not conform to the ONNX spec and contain initializers that are not graph inputs."); } + + node_arg_in_parent_graph = true; } // add it to the Node's list of implicit inputs - auto& node = *GetNode(nodeid_to_subgraphs.first); + auto& implicit_inputs = node->MutableDefinitions().implicit_input_defs; + if (std::find(implicit_inputs.cbegin(), implicit_inputs.cend(), node_arg) == implicit_inputs.cend()) { + implicit_inputs.push_back(node_arg); + } - node.MutableDefinitions().implicit_input_defs.push_back(node_arg); + if (node_arg_in_parent_graph || + resolve_context_.inputs_and_initializers.find(node_arg_name) != + resolve_context_.inputs_and_initializers.cend()) { + // no connection required if it's an input or initializer. + // if the node arg is from a parent graph we link the nodes in the parent graph by passing the + // node_arg_name back up in outer_scope_node_args_consumed - if (resolve_context_.inputs_and_initializers.find(node_arg_name) != - resolve_context_.inputs_and_initializers.cend()) { - // no connection required } else { // if it's an output nodearg in this graph we need to create a link to the node the output is coming from auto entry = resolve_context_.output_args.find(node_arg_name); @@ -881,7 +931,7 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c // Create relationship between this node (node), and the node providing the output (output_node). Node& output_node = *entry->second; - AddEdge(output_node.Index(), node.Index(), *node_arg); + AddEdge(output_node.Index(), node->Index(), *node_arg); inner_nodes.insert(&output_node); } @@ -1158,12 +1208,10 @@ class InferenceContextImpl : public ONNX_NAMESPACE::InferenceContext { public: InferenceContextImpl(Node& node, - const AttributeGraphMap* subgraphs = nullptr, - SubgraphInferencingFunc* subgraph_inferencing_func = nullptr, + SubgraphInferencingFunc subgraph_inferencing_func, const InitializedTensorSet& initialized_tensor_set = {}) noexcept : node_(node), - attr_to_subgraph_map_{subgraphs}, - subgraph_inferencing_func_{subgraph_inferencing_func}, + subgraph_inferencing_func_(subgraph_inferencing_func), initialized_tensor_set_(initialized_tensor_set) { node_output_types_.resize(node.OutputDefs().size()); } @@ -1223,17 +1271,15 @@ class InferenceContextImpl : public ONNX_NAMESPACE::InferenceContext { GraphInferencer* getGraphAttributeInferencer(const std::string& attribute_name) override { GraphInferencer* graph_inferencer = nullptr; - if (attr_to_subgraph_map_ && subgraph_inferencing_func_) { - auto attr_to_subgraph = attr_to_subgraph_map_->find(attribute_name); - if (attr_to_subgraph != attr_to_subgraph_map_->cend()) { - auto inferencer = std::make_unique(node_, *attr_to_subgraph->second, - *subgraph_inferencing_func_); - graph_inferencer = inferencer.get(); - graph_inferencers_.push_back(std::move(inferencer)); - } else { - fail_type_inference("No Graph instance was found for attribute ", - attribute_name, " in node ", node_.Name()); - } + auto* subgraph = node_.GetMutableGraphAttribute(attribute_name); + + if (subgraph) { + auto inferencer = std::make_unique(node_, *subgraph, subgraph_inferencing_func_); + graph_inferencer = inferencer.get(); + graph_inferencers_.push_back(std::move(inferencer)); + } else { + fail_type_inference("No Graph instance was found for attribute ", + attribute_name, " in node ", node_.Name()); } return graph_inferencer; @@ -1243,8 +1289,7 @@ class InferenceContextImpl : public ONNX_NAMESPACE::InferenceContext { Node& node_; // node_output_types_ will be populated by the operator-specific shape inference. std::vector node_output_types_; - const AttributeGraphMap* attr_to_subgraph_map_; - SubgraphInferencingFunc* subgraph_inferencing_func_; + SubgraphInferencingFunc subgraph_inferencing_func_; std::vector> graph_inferencers_; const InitializedTensorSet& initialized_tensor_set_; }; @@ -1404,9 +1449,7 @@ Status Graph::InferAndVerifyTypeMatch(Node& node, const OpSchema& op) { // Once that completes, the outputs from the node containing the subgraph will be updated, and the final values // returned here. SubgraphInferencingFunc func(Graph::InferAndVerifySubgraphTypes); - auto node_subgraphs = subgraph_map_.find(node.Index()); - auto* subgraphs = node_subgraphs != subgraph_map_.cend() ? &node_subgraphs->second : nullptr; - InferenceContextImpl context(node, subgraphs, &func, name_to_initial_tensor_); + InferenceContextImpl context(node, func, name_to_initial_tensor_); try { context.RunInferencing(); @@ -1572,8 +1615,8 @@ Status Graph::VerifyNodeAndOpMatch() { auto& node_name = node.Name(); auto& domain = node.Domain(); - auto iter = model_functions_.find(node.OpType()); - if (iter != model_functions_.end()) { + auto iter = model_functions_.find(node.OpType()); + if (iter != model_functions_.end()) { const ONNX_NAMESPACE::FunctionProto* model_function_proto = iter->second; auto model_func_ptr = std::make_unique(*this, node.Index(), model_function_proto); function_container_.emplace_back(std::move(model_func_ptr)); @@ -1647,74 +1690,13 @@ Status Graph::VerifyNodeAndOpMatch() { return Status::OK(); } -Graph* Graph::GetMutableSubgraph(const NodeIndex node_index, const std::string& attribute_name) { - const Graph* subgraph = GetSubgraph(node_index, attribute_name); - return const_cast(subgraph); -} - -const Graph* Graph::GetSubgraph(const NodeIndex node_index, const std::string& attribute_name) const { - Graph* subgraph = nullptr; - - auto entry = subgraph_map_.find(node_index); - - if (entry != subgraph_map_.cend()) { - auto& name_to_subgraph_map = entry->second; - auto subgraph_iter = name_to_subgraph_map.find(attribute_name); - if (subgraph_iter != name_to_subgraph_map.cend()) { - subgraph = subgraph_iter->second; - } - } - - return subgraph; -} - -Status Graph::CreateSubgraphs() { - Status status = Status::OK(); - - // don't use NodesInTopologicalOrder as we want CreateSubgraphs to recursively create subgraphs with no - // dependency on PerformTopologicalSortAndCheckIsAcyclic having been called previously - // to populate NodesInTopologicalOrder +void Graph::FindAllSubgraphs(std::vector& subgraphs) { for (auto& node : Nodes()) { - auto node_index = node.Index(); - if (subgraph_map_.find(node_index) != subgraph_map_.cend()) { - // if we have an existing entry we have processed this node previously. - // as the subgraph is loaded from a static GraphProto we assume nothing in - // it could have changed and there's no point re-creating it. - continue; - } - - // check attributes of all nodes looking for GraphProto attributes, and create - // the Graph instance for the subgraph contained in the GraphProto. - for (auto& attr : node.attributes_) { - bool has_subgraph = attr.second.has_g(); - if (has_subgraph) { - auto& attr_name = attr.first; - auto entry = subgraph_map_.find(node_index); - - // make sure this is new. internal logic error if it is not so using ONNXRUNTIME_ENFORCE. - if (entry != subgraph_map_.cend()) { - const auto& existing_entries = entry->second; - ONNXRUNTIME_ENFORCE(existing_entries.find(attr_name) == existing_entries.cend(), - "Entry exists in node ", node_index, " for attribute ", attr_name); - } - - auto& graph_proto = *attr.second.mutable_g(); - - // create instance. need to call private ctor so can't use make_unique - GSL_SUPPRESS(r .11) - std::unique_ptr subgraph{new Graph(*this, graph_proto)}; - - // Recursively create any further subgraphs - status = subgraph->CreateSubgraphs(); - ONNXRUNTIME_RETURN_IF_ERROR(status); - - subgraph_map_[node_index][attr_name] = subgraph.get(); - subgraphs_.push_back(std::move(subgraph)); - } + for (auto& subgraph : node.MutableSubgraphs()) { + subgraphs.push_back(subgraph.get()); + subgraph->FindAllSubgraphs(subgraphs); } } - - return Status::OK(); } Status Graph::VerifyInputAndInitializerNames() { @@ -1750,11 +1732,10 @@ Status Graph::InitInputsInitializersOutputs() { } // add the subgraph pointers to the resolve context. - for (auto& nodeid_to_subgraphs : subgraph_map_) { - resolve_context_.node_to_subgraphs_map[nodeid_to_subgraphs.first] = {}; - - for (auto& attr_name_to_subgraph : nodeid_to_subgraphs.second) { - resolve_context_.node_to_subgraphs_map[nodeid_to_subgraphs.first].push_back(attr_name_to_subgraph.second); + for (auto& node : Nodes()) { + auto& subgraphs = node.MutableSubgraphs(); + if (!subgraphs.empty()) { + resolve_context_.nodes_with_subgraphs.insert(&node); } } @@ -1785,14 +1766,15 @@ Status Graph::PerformTypeAndShapeInferencing() { return Status::OK(); } -Status Graph::ForThisAndAllSubgraphs(std::function func) { +Status Graph::ForThisAndAllSubgraphs(const std::vector& subgraphs, std::function func) { auto status = func(*this); ONNXRUNTIME_RETURN_IF_ERROR(status); - for (auto& subgraph : subgraphs_) { + for (auto& subgraph : subgraphs) { status = func(*subgraph); ONNXRUNTIME_RETURN_IF_ERROR(status); } + return status; } @@ -1808,8 +1790,12 @@ Status Graph::Resolve(bool no_proto_sync_required) { return status; } - bool subgraphs_need_resolve = std::any_of(subgraphs_.cbegin(), subgraphs_.cend(), - [](const std::unique_ptr& graph) { + // find all subgraphs including nested ones. + std::vector all_subgraphs; + FindAllSubgraphs(all_subgraphs); + + bool subgraphs_need_resolve = std::any_of(all_subgraphs.cbegin(), all_subgraphs.cend(), + [](const Graph* graph) { return graph->GraphResolveNeeded(); }); @@ -1817,14 +1803,9 @@ Status Graph::Resolve(bool no_proto_sync_required) { return Status::OK(); } - // Create the Graph instances for the subgraph/s in any nodes containing GraphProto attributes (Scan/If/Loop). - // Do this upfront so we can recurse into them when building connections and doing type/shape inferencing. - // Recursively creates any nested subgraphs. - ONNXRUNTIME_RETURN_IF_ERROR(CreateSubgraphs()); - // init all graph/subgraphs. non-recursive. auto init_func = [](Graph& graph) { return graph.InitInputsInitializersOutputs(); }; - ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(init_func)); + ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(all_subgraphs, init_func)); // recursively set the outer scope node args. ONNXRUNTIME_RETURN_IF_ERROR(SetOuterScopeNodeArgs(resolve_context_.outer_scope_node_args)); @@ -1838,7 +1819,7 @@ Status Graph::Resolve(bool no_proto_sync_required) { // topological sort of this and any subgraphs is non-recursive auto topo_sort_func = [](Graph& graph) { return graph.PerformTopologicalSortAndCheckIsAcyclic(); }; - ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(topo_sort_func)); + ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(all_subgraphs, topo_sort_func)); // type/shape validation and inferencing on this and any subgraphs // recurses into subgraphs via the ONNX checker, which descends into the GraphProto in node attributes @@ -1858,7 +1839,7 @@ Status Graph::Resolve(bool no_proto_sync_required) { return Status::OK(); }; - ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(finalize_func)); + ONNXRUNTIME_RETURN_IF_ERROR(ForThisAndAllSubgraphs(all_subgraphs, finalize_func)); return Status::OK(); } @@ -2166,6 +2147,10 @@ void Graph::CleanUnusedInitializers() { node.ForEachInputDef([&used_args](const onnxruntime::NodeArg* def) { ONNXRUNTIME_IGNORE_RETURN_VALUE(used_args.insert(def->Name())); }); + + for (const auto* def : node.ImplicitInputDefs()) { + ONNXRUNTIME_IGNORE_RETURN_VALUE(used_args.insert(def->Name())); + } } std::vector erase_list; diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index b03cd4c84c68e..bdefbf46adeca 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -263,6 +263,17 @@ struct MLAS_PLATFORM { extern MLAS_PLATFORM MlasPlatform; +// +// Define the missing ARM64 NEON intrinsic macros from arm64_neon.h that enable +// cross-compiler support. +// + +#if defined(_M_ARM64) +#ifndef vmaxvq_f32 +#define vmaxvq_f32(src) neon_fmaxv(src) +#endif +#endif + // // Cross-platform wrappers for vector intrinsics. // diff --git a/onnxruntime/core/mlas/lib/pooling.cpp b/onnxruntime/core/mlas/lib/pooling.cpp index f535d339d2ca2..73eae593753d6 100644 --- a/onnxruntime/core/mlas/lib/pooling.cpp +++ b/onnxruntime/core/mlas/lib/pooling.cpp @@ -24,6 +24,7 @@ Module Name: struct MLAS_WORK_BLOCK { MLAS_POOLING_KIND PoolingKind; size_t InputShape[3]; + size_t InputSize; size_t OutputShape[3]; int64_t KernelShape[3]; int64_t Padding[6]; @@ -86,6 +87,22 @@ struct MLAS_MAXIMUM_POOLING return MlasMaximumFloat32x4(Reduction, Value); } +#if defined(MLAS_NEON64_INTRINSICS) + + static float ReduceFloat32x4(MLAS_FLOAT32X4 Reduction) + { + return vmaxvq_f32(Reduction); + } + +#elif defined(MLAS_NEON32_INTRINSICS) + + static float32x2_t ReducePairwise(float32x2_t Vector0, float32x2_t Vector1) + { + return vpmax_f32(Vector0, Vector1); + } + +#endif + static float AveragePool(float Reduction, float Size) { MLAS_UNREFERENCED_PARAMETER(Size); @@ -152,6 +169,25 @@ struct MLAS_AVERAGE_POOLING return MlasAddFloat32x4(Reduction, Value); } +#if defined(MLAS_NEON64_INTRINSICS) + + static float ReduceFloat32x4(MLAS_FLOAT32X4 Reduction) + { + Reduction = vpaddq_f32(Reduction, Reduction); + Reduction = vpaddq_f32(Reduction, Reduction); + + return vgetq_lane_f32(Reduction, 0); + } + +#elif defined(MLAS_NEON32_INTRINSICS) + + static float32x2_t ReducePairwise(float32x2_t Vector0, float32x2_t Vector1) + { + return vpadd_f32(Vector0, Vector1); + } + +#endif + static float AveragePool(float Reduction, float Size) { return Reduction / Size; @@ -340,6 +376,7 @@ Return Value: const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t InputSize = WorkBlock->InputSize; const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; @@ -386,7 +423,7 @@ Return Value: } } - Input += InputHeight * InputWidth; + Input += InputSize; } } @@ -430,6 +467,7 @@ Return Value: const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t InputSize = WorkBlock->InputSize; const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; @@ -609,7 +647,7 @@ Return Value: } while (OutputWidthRemaining > 0); } - Input += InputHeight * InputWidth; + Input += InputSize; } } @@ -652,6 +690,7 @@ Return Value: const size_t InputDepth = WorkBlock->InputShape[DepthShapeIndex]; const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t InputSize = WorkBlock->InputSize; const size_t OutputDepth = WorkBlock->OutputShape[DepthShapeIndex]; const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; @@ -713,7 +752,7 @@ Return Value: } } - Input += InputDepth * InputHeight * InputWidth; + Input += InputSize; } } @@ -759,6 +798,7 @@ Return Value: const size_t InputDepth = WorkBlock->InputShape[DepthShapeIndex]; const size_t InputHeight = WorkBlock->InputShape[HeightShapeIndex]; const size_t InputWidth = WorkBlock->InputShape[WidthShapeIndex]; + const size_t InputSize = WorkBlock->InputSize; const size_t OutputDepth = WorkBlock->OutputShape[DepthShapeIndex]; const size_t OutputHeight = WorkBlock->OutputShape[HeightShapeIndex]; const size_t OutputWidth = WorkBlock->OutputShape[WidthShapeIndex]; @@ -980,7 +1020,108 @@ Return Value: } } - Input += InputDepth * InputHeight * InputWidth; + Input += InputSize; + } +} + +template +void +MlasPoolGlobalKernel( + const MLAS_WORK_BLOCK* WorkBlock, + size_t ChannelCount, + const float* Input, + float* Output + ) +/*++ + +Routine Description: + + This routine implements a global pooling operation. + +Arguments: + + WorkBlock - Supplies the structure that contains the pooling parameters. + + ChannelCount - Supplies the number of channels to process. + + Input - Supplies the input tensor. + + Output - Supplies the output tensor. + +Return Value: + + None. + +--*/ +{ + const size_t InputSize = WorkBlock->InputSize; + const float InputSizeFloat = float(InputSize); + + // + // Apply the pooling operation to each channel. + // + + for (size_t c = 0; c < ChannelCount; c++) { + + size_t InputSizeRemaining = InputSize; + + // + // Iterate over the input buffer a vector at a time. + // + + MLAS_FLOAT32X4 Reduction = PoolingType::InitialVector(); + + while (InputSizeRemaining >= 4) { + Reduction = PoolingType::Reduce(Reduction, MlasLoadFloat32x4(Input)); + Input += 4; + InputSizeRemaining -= 4; + } + + // + // Reduce the vector to a single float value. + // + +#if defined(MLAS_NEON64_INTRINSICS) + + float ReductionValue = PoolingType::ReduceFloat32x4(Reduction); + +#elif defined(MLAS_NEON32_INTRINSICS) + + float32x2_t ReductionLow = vget_low_f32(Reduction); + float32x2_t ReductionHigh = vget_high_f32(Reduction); + + ReductionLow = PoolingType::ReducePairwise(ReductionLow, ReductionHigh); + ReductionLow = PoolingType::ReducePairwise(ReductionLow, ReductionHigh); + + float ReductionValue = vget_lane_f32(ReductionLow, 0); + +#elif defined(MLAS_SSE2_INTRINSICS) + + Reduction = PoolingType::Reduce(Reduction, _mm_shuffle_ps(Reduction, Reduction, _MM_SHUFFLE(3, 2, 3, 2))); + Reduction = PoolingType::Reduce(Reduction, _mm_shuffle_ps(Reduction, Reduction, _MM_SHUFFLE(1, 1, 1, 1))); + + float ReductionValue = _mm_cvtss_f32(Reduction); + +#else +#error Unsupported architecture. +#endif + + // + // Iterate over the remaining input buffer an element at a time. + // + + while (InputSizeRemaining > 0) { + ReductionValue = PoolingType::Reduce(ReductionValue, *Input++); + InputSizeRemaining -= 1; + } + + // + // Apply average pooling if necessary. + // + + ReductionValue = PoolingType::AveragePool(ReductionValue, InputSizeFloat); + + *Output++ = ReductionValue; } } @@ -1007,6 +1148,13 @@ static const PMLAS_POOL_KERNEL_ROUTINE MlasPoolGenericKernels[][3] = }, }; +static const PMLAS_POOL_KERNEL_ROUTINE MlasPoolGlobalKernels[] = +{ + MlasPoolGlobalKernel, + MlasPoolGlobalKernel, + MlasPoolGlobalKernel, +}; + static const PMLAS_POOL_KERNEL_ROUTINE MlasPoolVectorKernels[][2] = { { @@ -1040,7 +1188,7 @@ MlasPool( Routine Description: - This routine prepares for a pooling operation by computing required WorkBlock. + This routine implements the pooling operation. Arguments: @@ -1090,6 +1238,9 @@ Return Value: size_t InputSize = 1; size_t OutputSize = 1; + bool InputAndKernelShapeMatch = true; + bool AllStridesAreOne = true; + bool AllPaddingIsZero = true; bool AllKernelsAreSmall = true; for (size_t dim = 0; dim < Dimensions; dim++) { @@ -1120,9 +1271,14 @@ Return Value: InputSize *= WorkBlock.InputShape[dim]; OutputSize *= WorkBlock.OutputShape[dim]; + InputAndKernelShapeMatch &= (WorkBlock.KernelShape[dim] == int64_t(WorkBlock.InputShape[dim])); + AllStridesAreOne &= (WorkBlock.StrideShape[dim] == 1); + AllPaddingIsZero &= (WorkBlock.Padding[dim] == 0 && WorkBlock.Padding[dim + Dimensions] == 0); AllKernelsAreSmall &= (WorkBlock.KernelShape[dim] <= 32); } + WorkBlock.InputSize = InputSize; + // // Determine which pooling kernel routine to use. // @@ -1134,7 +1290,11 @@ Return Value: PMLAS_POOL_KERNEL_ROUTINE PoolKernelRoutine = MlasPoolGenericKernels[PoolingKind][Dimensions - 1]; - if (Dimensions >= 2 && WorkBlock.StrideShape[Dimensions - 1] <= 2 && AllKernelsAreSmall) { + if (InputAndKernelShapeMatch && AllStridesAreOne && AllPaddingIsZero) { + + PoolKernelRoutine = MlasPoolGlobalKernels[PoolingKind]; + + } else if (Dimensions >= 2 && WorkBlock.StrideShape[Dimensions - 1] <= 2 && AllKernelsAreSmall) { int64_t ReductionBufferRemaining = MLAS_POOL_REDUCTION_BUFFER_STACK - MLAS_POOL_REDUCTION_BUFFER_PADDING; diff --git a/onnxruntime/core/mlas/lib/sgemm.cpp b/onnxruntime/core/mlas/lib/sgemm.cpp index 11f0d22121260..66b53c6041138 100644 --- a/onnxruntime/core/mlas/lib/sgemm.cpp +++ b/onnxruntime/core/mlas/lib/sgemm.cpp @@ -147,9 +147,9 @@ Routine Description: lda - Supplies the number of elements per row of the source matrix. - CountY - Supplies the number of rows of the source matrix to transpose. + CountY - Supplies the number of columns of the source matrix to transpose. - CountX - Supplies the number of columns of the source matrix to transpose. + CountX - Supplies the number of rows of the source matrix to transpose. Return Value: diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc index ceaaf8f97b0cb..bfc134b0f0064 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc @@ -311,315 +311,6 @@ ONNX_CPU_OPERATOR_KERNEL( KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Scale); -template -auto MakeEigenArrayMap(Tensor& t) { return EigenVectorArrayMap(t.template MutableData(), t.Shape().Size()); } -template -auto MakeEigenArrayMap(const Tensor& t) { return ConstEigenVectorArrayMap(t.template Data(), t.Shape().Size()); } - -struct BroadcastIterator { - size_t AdvanceBy(size_t delta) { - size_t index = index_; - - index_ += deltas_[0] * delta; - counters_[0] += delta; - if (counters_[0] == counts_[0]) { - counters_[0] = 0; - for (size_t counterIndex = 1; counterIndex < counters_.size(); counterIndex++) { - index_ += deltas_[counterIndex]; - if (++counters_[counterIndex] != counts_[counterIndex]) - break; - counters_[counterIndex] = 0; - } - } - return index; - } - - void Init(int64_t axis, int64_t largest) { - ONNXRUNTIME_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); - - deltas_.push_back(axis > 1); - counts_.push_back(largest); - count_ *= axis; - } - - void Append(int64_t axis, int64_t largest) { - ONNXRUNTIME_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); - - // If we're greater than 1, it doesn't matter what the other tensor does - if (axis > 1) { - if (deltas_.back() <= 0) // Were we broadcasting - StopBroadcasting(); - } else { // We must be 1, at this point - if (deltas_.back() > 0) - StartBroadcasting(); - } - - counts_.back() *= largest; // Just increase the last count - count_ *= axis; - } - - void StopBroadcasting() { - deltas_.push_back(count_); - counts_.push_back(1); - } - - void StartBroadcasting() { - deltas_.push_back(-count_); - counts_.push_back(1); - } - - std::vector counters_; - std::vector deltas_; - std::vector counts_; - size_t count_{1}; // Running total count of entries in tensor, used while building up the entries - - private: - size_t index_{}; -}; - -struct Broadcaster { - Broadcaster(const std::vector& shape1, const std::vector& shape2) { - size_t dimension_count_max = std::max(shape1.size(), shape2.size()); - size_t dimension_count_min = std::min(shape1.size(), shape2.size()); - output_shape_.resize(dimension_count_max); - - auto iter1 = shape1.end(); - auto iter2 = shape2.end(); - auto output_shape = output_shape_.end(); - - // Scalars are a special case, as it's always a broadcast - size_t index = 0; - if (dimension_count_min == 0) { - if (shape1.size() == 0) // Shape1 is a scalar - { - if (shape2.size() == 0) // Two scalars? - { - iterator1_.Init(1, 1); - iterator2_.Init(1, 1); - } else { - auto axis = *--iter2; - iterator1_.Init(1, axis); - iterator2_.Init(axis, axis); - *--output_shape = axis; - } - } else { // Shape2 is a scalar - auto axis = *--iter1; - iterator1_.Init(axis, axis); - iterator2_.Init(1, axis); - *--output_shape = axis; - } - index++; // Manually increment since we processed one axis - } - - for (; index < dimension_count_min; index++) { - auto axis1 = *--iter1; - auto axis2 = *--iter2; - - auto largest = std::max(axis1, axis2); - *--output_shape = largest; - - if (largest == 1 && index + 1 < dimension_count_min) // Nothing to do in this case - continue; - - iterator1_.Init(axis1, largest); - iterator2_.Init(axis2, largest); - index++; // Manually increment since we processed one axis - break; - } - - for (; index < dimension_count_min; index++) { - auto axis1 = *--iter1; - auto axis2 = *--iter2; - - auto largest = std::max(axis1, axis2); - *--output_shape = largest; - - if (largest == 1) // Nothing to do in this case - continue; - - iterator1_.Append(axis1, largest); - iterator2_.Append(axis2, largest); - } - - // If one shape is bigger than another we need to broadcast the smaller onto the bigger from this point on - for (; index < dimension_count_max; index++) { - if (dimension_count_max == shape2.size()) { - auto axis = *--iter2; - iterator1_.Append(1, axis); - iterator2_.Append(axis, axis); - *--output_shape = axis; - } else { - auto axis = *--iter1; - iterator1_.Append(axis, axis); - iterator2_.Append(1, axis); - *--output_shape = axis; - } - } - - // Allocate the counters - iterator1_.counters_.resize(iterator1_.counts_.size(), 0); - iterator2_.counters_.resize(iterator2_.counts_.size(), 0); - } - - size_t GetSpanSize() const { return std::min(iterator1_.counts_.front(), iterator2_.counts_.front()); } - - BroadcastIterator iterator1_, iterator2_; - std::vector output_shape_; -}; - -template -struct TBroadcaster { - TBroadcaster(const Tensor& input0, const Tensor& input1) - : input_tensor0_(input0), - input_tensor1_(input1) { - } - - TensorShape GetOutputShape() const { return TensorShape(broadcaster_.output_shape_); } - size_t GetSpanSize() const { return span_size_; } - - bool IsInput0Scalar() const { return broadcaster_.iterator1_.deltas_.front() == 0; } - bool IsInput1Scalar() const { return broadcaster_.iterator2_.deltas_.front() == 0; } - - T NextScalar0() { return *Next0(); } - T NextScalar1() { return *Next1(); } - - gsl::span NextSpan0() { return gsl::span(Next0(), span_size_); } - gsl::span NextSpan1() { return gsl::span(Next1(), span_size_); } - - ConstEigenVectorMap NextEigen0() { return ConstEigenVectorMap(Next0(), span_size_); } - ConstEigenVectorMap NextEigen1() { return ConstEigenVectorMap(Next1(), span_size_); } - - private: - const T* Next0() { return input0_ + broadcaster_.iterator1_.AdvanceBy(span_size_); } - const T* Next1() { return input1_ + broadcaster_.iterator2_.AdvanceBy(span_size_); } - - const Tensor& input_tensor0_; - const Tensor& input_tensor1_; - Broadcaster broadcaster_{input_tensor0_.Shape().GetDims(), input_tensor1_.Shape().GetDims()}; - size_t span_size_{broadcaster_.GetSpanSize()}; - - const T* input0_{input_tensor0_.template Data()}; - const T* input1_{input_tensor1_.template Data()}; -}; - -template -struct TBroadcastOutput { - TBroadcastOutput(size_t span_size, Tensor& tensor) - : span_size_(span_size) { - output_ = tensor.template MutableData(); - output_end_ = output_ + tensor.Shape().Size(); - } - - operator bool() const { - return output_ != output_end_; - } - - EigenVectorMap NextEigenOutput() { - return EigenVectorMap(NextOutput(), span_size_); - } - - gsl::span NextSpanOutput() { - return gsl::span(NextOutput(), span_size_); - } - - private: - T* NextOutput() { - T* output = output_; - output_ += span_size_; - return output; - } - - T* output_; - const T* output_end_; - size_t span_size_; -}; - -template -struct TensorAllocator { - TensorAllocator(OpKernelContext& context) { - ONNXRUNTIME_ENFORCE(context.GetTempSpaceAllocator(&allocator_).IsOK()); - } - - std::unique_ptr Allocate(const TensorShape& shape) { - return std::make_unique(DataTypeImpl::GetType(), - shape, - allocator_->Alloc(sizeof(T) * shape.Size()), - allocator_->Info(), - allocator_); - } - - private: - AllocatorPtr allocator_; -}; - -// Broadcast loop for when using eigen, functions are in this form: -// Input0Scalar: [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) -// Input1Scalar: [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) -// General : [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) -template -void BroadcastLoop(TBroadcaster& bc, Output& output, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { - if (bc.IsInput0Scalar()) { - while (output) - input0scalar(output.NextEigenOutput(), bc.NextScalar0(), bc.NextEigen1()); - } else if (bc.IsInput1Scalar()) { - while (output) - input1scalar(output.NextEigenOutput(), bc.NextEigen0(), bc.NextScalar1()); - } else { - while (output) - general(output.NextEigenOutput(), bc.NextEigen0(), bc.NextEigen1()); - } -} - -template -Status BroadcastTwo(OpKernelContext& context, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { - TBroadcaster bc(*context.Input(0), *context.Input(1)); - TBroadcastOutput output(bc.GetSpanSize(), *context.Output(0, bc.GetOutputShape())); - BroadcastLoop(bc, output, input0scalar, input1scalar, general); - - return Status::OK(); -} - -template -Status BroadcastVariadic(const Node& node, OpKernelContext& context, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { - auto input_count = node.InputArgCount().front(); - ONNXRUNTIME_ENFORCE(input_count >= 1, "Must have 1 or more inputs"); - - // One item is trivial, just copy across and exit - if (input_count == 1) { - EigenMap(*context.Output(0, context.Input(0)->Shape())) = EigenMap(*context.Input(0)); - return Status::OK(); - } - - std::unique_ptr tempInput; - std::unique_ptr tempOutput; - - TensorAllocator tensorAllocator(context); - - // For more than 2 tensors, we sum the first two into a temporary tensor, then sum the next with the temporary tensor - for (int i = 0; i < input_count - 1; i++) { - auto& tensor0 = tempInput ? *tempInput : *context.Input(0); - auto& tensor1 = *context.Input(i + 1); - - TBroadcaster bc(tensor0, tensor1); - - // Create a temporary output for all but the last iteration, which goes to the real output - Tensor* p_output{}; - if (i == input_count - 2) - p_output = context.Output(0, bc.GetOutputShape()); - else { - tempOutput = tensorAllocator.Allocate(bc.GetOutputShape()); - p_output = tempOutput.get(); - } - - TBroadcastOutput output(bc.GetSpanSize(), *p_output); - - BroadcastLoop(bc, output, input0scalar, input1scalar, general); - - tempInput = std::move(tempOutput); - } - return Status::OK(); -} - template Status Add::Compute(OpKernelContext* context) const { return BroadcastTwo( diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.h b/onnxruntime/core/providers/cpu/math/element_wise_ops.h index 13d9c5ea5df52..a0b453c183a36 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.h +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.h @@ -317,4 +317,314 @@ class Scale final : public OpKernel { float scale_; }; +template +auto MakeEigenArrayMap(Tensor& t) { return EigenVectorArrayMap(t.template MutableData(), t.Shape().Size()); } +template +auto MakeEigenArrayMap(const Tensor& t) { return ConstEigenVectorArrayMap(t.template Data(), t.Shape().Size()); } + +struct BroadcastIterator { + size_t AdvanceBy(size_t delta) { + size_t index = index_; + + index_ += deltas_[0] * delta; + counters_[0] += delta; + if (counters_[0] == counts_[0]) { + counters_[0] = 0; + for (size_t counterIndex = 1; counterIndex < counters_.size(); counterIndex++) { + index_ += deltas_[counterIndex]; + if (++counters_[counterIndex] != counts_[counterIndex]) + break; + counters_[counterIndex] = 0; + } + } + return index; + } + + void Init(int64_t axis, int64_t largest) { + ONNXRUNTIME_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); + + deltas_.push_back(axis > 1); + counts_.push_back(largest); + count_ *= axis; + } + + void Append(int64_t axis, int64_t largest) { + ONNXRUNTIME_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); + + // If we're greater than 1, it doesn't matter what the other tensor does + if (axis > 1) { + if (deltas_.back() <= 0) // Were we broadcasting + StopBroadcasting(); + } else { // We must be 1, at this point + if (deltas_.back() > 0) + StartBroadcasting(); + } + + counts_.back() *= largest; // Just increase the last count + count_ *= axis; + } + + void StopBroadcasting() { + deltas_.push_back(count_); + counts_.push_back(1); + } + + void StartBroadcasting() { + deltas_.push_back(-count_); + counts_.push_back(1); + } + + std::vector counters_; + std::vector deltas_; + std::vector counts_; + size_t count_{1}; // Running total count of entries in tensor, used while building up the entries + + private: + size_t index_{}; +}; + +struct Broadcaster { + Broadcaster(const std::vector& shape1, const std::vector& shape2) { + size_t dimension_count_max = std::max(shape1.size(), shape2.size()); + size_t dimension_count_min = std::min(shape1.size(), shape2.size()); + output_shape_.resize(dimension_count_max); + + auto iter1 = shape1.end(); + auto iter2 = shape2.end(); + auto output_shape = output_shape_.end(); + + // Scalars are a special case, as it's always a broadcast + size_t index = 0; + if (dimension_count_min == 0) { + if (shape1.size() == 0) // Shape1 is a scalar + { + if (shape2.size() == 0) // Two scalars? + { + iterator1_.Init(1, 1); + iterator2_.Init(1, 1); + } else { + auto axis = *--iter2; + iterator1_.Init(1, axis); + iterator2_.Init(axis, axis); + *--output_shape = axis; + } + } else { // Shape2 is a scalar + auto axis = *--iter1; + iterator1_.Init(axis, axis); + iterator2_.Init(1, axis); + *--output_shape = axis; + } + index++; // Manually increment since we processed one axis + } + + for (; index < dimension_count_min; index++) { + auto axis1 = *--iter1; + auto axis2 = *--iter2; + + auto largest = std::max(axis1, axis2); + *--output_shape = largest; + + if (largest == 1 && index + 1 < dimension_count_min) // Nothing to do in this case + continue; + + iterator1_.Init(axis1, largest); + iterator2_.Init(axis2, largest); + index++; // Manually increment since we processed one axis + break; + } + + for (; index < dimension_count_min; index++) { + auto axis1 = *--iter1; + auto axis2 = *--iter2; + + auto largest = std::max(axis1, axis2); + *--output_shape = largest; + + if (largest == 1) // Nothing to do in this case + continue; + + iterator1_.Append(axis1, largest); + iterator2_.Append(axis2, largest); + } + + // If one shape is bigger than another we need to broadcast the smaller onto the bigger from this point on + for (; index < dimension_count_max; index++) { + if (dimension_count_max == shape2.size()) { + auto axis = *--iter2; + iterator1_.Append(1, axis); + iterator2_.Append(axis, axis); + *--output_shape = axis; + } else { + auto axis = *--iter1; + iterator1_.Append(axis, axis); + iterator2_.Append(1, axis); + *--output_shape = axis; + } + } + + // Allocate the counters + iterator1_.counters_.resize(iterator1_.counts_.size(), 0); + iterator2_.counters_.resize(iterator2_.counts_.size(), 0); + } + + size_t GetSpanSize() const { return std::min(iterator1_.counts_.front(), iterator2_.counts_.front()); } + + BroadcastIterator iterator1_, iterator2_; + std::vector output_shape_; +}; + +template +struct TBroadcaster { + TBroadcaster(const Tensor& input0, const Tensor& input1) + : input_tensor0_(input0), + input_tensor1_(input1) { + } + + TensorShape GetOutputShape() const { return TensorShape(broadcaster_.output_shape_); } + size_t GetSpanSize() const { return span_size_; } + + bool IsInput0Scalar() const { return broadcaster_.iterator1_.deltas_.front() == 0; } + bool IsInput1Scalar() const { return broadcaster_.iterator2_.deltas_.front() == 0; } + + T NextScalar0() { return *Next0(); } + T NextScalar1() { return *Next1(); } + + gsl::span NextSpan0() { return gsl::span(Next0(), span_size_); } + gsl::span NextSpan1() { return gsl::span(Next1(), span_size_); } + + ConstEigenVectorMap NextEigen0() { return ConstEigenVectorMap(Next0(), span_size_); } + ConstEigenVectorMap NextEigen1() { return ConstEigenVectorMap(Next1(), span_size_); } + + private: + const T* Next0() { return input0_ + broadcaster_.iterator1_.AdvanceBy(span_size_); } + const T* Next1() { return input1_ + broadcaster_.iterator2_.AdvanceBy(span_size_); } + + const Tensor& input_tensor0_; + const Tensor& input_tensor1_; + Broadcaster broadcaster_{input_tensor0_.Shape().GetDims(), input_tensor1_.Shape().GetDims()}; + size_t span_size_{broadcaster_.GetSpanSize()}; + + const T* input0_{input_tensor0_.template Data()}; + const T* input1_{input_tensor1_.template Data()}; +}; + +template +struct TBroadcastOutput { + TBroadcastOutput(size_t span_size, Tensor& tensor) + : span_size_(span_size) { + output_ = tensor.template MutableData(); + output_end_ = output_ + tensor.Shape().Size(); + } + + operator bool() const { + return output_ != output_end_; + } + + EigenVectorMap NextEigenOutput() { + return EigenVectorMap(NextOutput(), span_size_); + } + + gsl::span NextSpanOutput() { + return gsl::span(NextOutput(), span_size_); + } + + private: + T* NextOutput() { + T* output = output_; + output_ += span_size_; + return output; + } + + T* output_; + const T* output_end_; + size_t span_size_; +}; + +template +struct TensorAllocator { + TensorAllocator(OpKernelContext& context) { + ONNXRUNTIME_ENFORCE(context.GetTempSpaceAllocator(&allocator_).IsOK()); + } + + std::unique_ptr Allocate(const TensorShape& shape) { + return std::make_unique(DataTypeImpl::GetType(), + shape, + allocator_->Alloc(sizeof(T) * shape.Size()), + allocator_->Info(), + allocator_); + } + + private: + AllocatorPtr allocator_; +}; + +// Broadcast loop for when using eigen, functions are in this form: +// Input0Scalar: [](EigenVectorMap output, T input0, ConstEigenVectorMap input1) +// Input1Scalar: [](EigenVectorMap output, ConstEigenVectorMap input0, T input1) +// General : [](EigenVectorMap output, ConstEigenVectorMap input0, ConstEigenVectorMap input1) +template +void BroadcastLoop(TBroadcaster& bc, Output& output, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { + if (bc.IsInput0Scalar()) { + while (output) + input0scalar(output.NextEigenOutput(), bc.NextScalar0(), bc.NextEigen1()); + } else if (bc.IsInput1Scalar()) { + while (output) + input1scalar(output.NextEigenOutput(), bc.NextEigen0(), bc.NextScalar1()); + } else { + while (output) + general(output.NextEigenOutput(), bc.NextEigen0(), bc.NextEigen1()); + } +} + +template +Status BroadcastTwo(OpKernelContext& context, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { + TBroadcaster bc(*context.Input(0), *context.Input(1)); + TBroadcastOutput output(bc.GetSpanSize(), *context.Output(0, bc.GetOutputShape())); + BroadcastLoop(bc, output, input0scalar, input1scalar, general); + + return Status::OK(); +} + +template +Status BroadcastVariadic(const Node& node, OpKernelContext& context, Input0Scalar input0scalar, Input1Scalar input1scalar, General general) { + auto input_count = node.InputArgCount().front(); + ONNXRUNTIME_ENFORCE(input_count >= 1, "Must have 1 or more inputs"); + + // One item is trivial, just copy across and exit + if (input_count == 1) { + EigenMap(*context.Output(0, context.Input(0)->Shape())) = EigenMap(*context.Input(0)); + return Status::OK(); + } + + std::unique_ptr tempInput; + std::unique_ptr tempOutput; + + TensorAllocator tensorAllocator(context); + + // For more than 2 tensors, we sum the first two into a temporary tensor, then sum the next with the temporary tensor + for (int i = 0; i < input_count - 1; i++) { + auto& tensor0 = tempInput ? *tempInput : *context.Input(0); + auto& tensor1 = *context.Input(i + 1); + + TBroadcaster bc(tensor0, tensor1); + + // Create a temporary output for all but the last iteration, which goes to the real output + Tensor* p_output{}; + if (i == input_count - 2) + p_output = context.Output(0, bc.GetOutputShape()); + else { + tempOutput = tensorAllocator.Allocate(bc.GetOutputShape()); + p_output = tempOutput.get(); + } + + TBroadcastOutput output(bc.GetSpanSize(), *p_output); + + BroadcastLoop(bc, output, input0scalar, input1scalar, general); + + tempInput = std::move(tempOutput); + } + return Status::OK(); +} + + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index 723523d7b24d0..f8c7a1349175d 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -1,9 +1,13 @@ ONNXRuntimeAddCustomOp ONNXRuntimeAddRefToObject +ONNXRuntimeAllocatorAlloc +ONNXRuntimeAllocatorFree +ONNXRuntimeAllocatorGetInfo ONNXRuntimeAllocatorInfoGetId ONNXRuntimeAllocatorInfoGetMemType ONNXRuntimeAllocatorInfoGetName ONNXRuntimeAllocatorInfoGetType +ONNXRuntimeCastTypeInfoToTensorInfo ONNXRuntimeCloneSessionOptions ONNXRuntimeCompareAllocatorInfo ONNXRuntimeCreateAllocatorInfo @@ -35,21 +39,24 @@ ONNXRuntimeGetTensorElementType ONNXRuntimeGetTensorMutableData ONNXRuntimeGetTensorShapeAndType ONNXRuntimeGetTensorShapeElementCount +ONNXRuntimeGetTypeInfo +ONNXRuntimeGetValueType ONNXRuntimeInferenceSessionGetInputCount ONNXRuntimeInferenceSessionGetInputName +ONNXRuntimeInferenceSessionGetInputTypeInfo ONNXRuntimeInferenceSessionGetOutputCount ONNXRuntimeInferenceSessionGetOutputName +ONNXRuntimeInferenceSessionGetOutputTypeInfo ONNXRuntimeInitialize ONNXRuntimeInitializeWithCustomLogger ONNXRuntimeIsTensor -ONNXRuntimeONNXValueListGetNthValue ONNXRuntimeReleaseObject ONNXRuntimeRunInference -ONNXRuntimeRunInferenceAndFetchAll ONNXRuntimeRunOptionsGetRunLogVerbosityLevel ONNXRuntimeRunOptionsGetRunTag ONNXRuntimeRunOptionsSetRunLogVerbosityLevel ONNXRuntimeRunOptionsSetRunTag +ONNXRuntimeRunOptionsSetTerminate ONNXRuntimeSessionOptionsAppendExecutionProvider ONNXRuntimeSetDims ONNXRuntimeSetSessionLogId @@ -62,7 +69,3 @@ ReleaseONNXRuntimeAllocatorInfo ReleaseONNXSession ReleaseONNXStatus ReleaseONNXValue -ReleaseONNXValueList -ONNXRuntimeAllocatorAlloc -ONNXRuntimeAllocatorFree -ONNXRuntimeAllocatorGetInfo \ No newline at end of file diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index bed078ec5faa6..1c8adbf133d67 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -3,31 +3,14 @@ #include "core/session/onnxruntime_c_api.h" #include +#include #include "core/session/inference_session.h" #include "abi_session_options_impl.h" -uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseCuda(void* this_) { - ONNXRuntimeSessionOptions* this_ptr = static_cast(this_); - if (--this_ptr->ref_count == 0) - delete this_ptr; - return 0; -} -uint32_t ONNXRUNTIME_API_STATUSCALL AddRefCuda(void* this_) { - ONNXRuntimeSessionOptions* this_ptr = static_cast(this_); - ++this_ptr->ref_count; - return 0; -} - -constexpr ONNXObject mkl_cls = { - AddRefCuda, - ReleaseCuda, -}; - -ONNXRuntimeSessionOptions::ONNXRuntimeSessionOptions() : cls(&mkl_cls), ref_count(1) { -} ONNXRuntimeSessionOptions::~ONNXRuntimeSessionOptions() { + assert(ref_count == 0); for (ONNXRuntimeProviderFactoryPtr* p : provider_factories) { ONNXRuntimeReleaseObject(p); } @@ -37,7 +20,7 @@ ONNXRuntimeSessionOptions& ONNXRuntimeSessionOptions::operator=(const ONNXRuntim throw std::runtime_error("not implemented"); } ONNXRuntimeSessionOptions::ONNXRuntimeSessionOptions(const ONNXRuntimeSessionOptions& other) - : cls(&mkl_cls), ref_count(1), value(other.value), custom_op_paths(other.custom_op_paths), provider_factories(other.provider_factories) { + : value(other.value), custom_op_paths(other.custom_op_paths), provider_factories(other.provider_factories) { for (ONNXRuntimeProviderFactoryPtr* p : other.provider_factories) { ONNXRuntimeAddRefToObject(p); } diff --git a/onnxruntime/core/session/abi_session_options_impl.h b/onnxruntime/core/session/abi_session_options_impl.h index ea0ba68ce14d1..6cf324acd6e82 100644 --- a/onnxruntime/core/session/abi_session_options_impl.h +++ b/onnxruntime/core/session/abi_session_options_impl.h @@ -6,16 +6,15 @@ #include #include #include +#include "core/framework/onnx_object_cxx.h" #include "core/session/inference_session.h" #include "core/session/onnxruntime_c_api.h" -struct ONNXRuntimeSessionOptions { - const ONNXObject* const cls; - std::atomic_int ref_count; +struct ONNXRuntimeSessionOptions : public onnxruntime::ObjectBase { onnxruntime::SessionOptions value; std::vector custom_op_paths; std::vector provider_factories; - ONNXRuntimeSessionOptions(); + ONNXRuntimeSessionOptions() = default; ~ONNXRuntimeSessionOptions(); ONNXRuntimeSessionOptions(const ONNXRuntimeSessionOptions& other); ONNXRuntimeSessionOptions& operator=(const ONNXRuntimeSessionOptions& other); diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 31b1400d25cd8..e8b580ae31155 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -261,7 +261,7 @@ class InferenceSession::Impl { // check if it has a subgraph if (proto.has_g()) { - Graph* subgraph = graph.GetMutableSubgraph(node.Index(), name); + Graph* subgraph = node.GetMutableGraphAttribute(name); ONNXRUNTIME_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); SubgraphMemory subgraph_info; @@ -273,8 +273,10 @@ class InferenceSession::Impl { SessionStateInitializer initializer{*subgraph, *subgraph_info.session_state, execution_providers_, kernel_registry_manager_, *session_logger_}; - ONNXRUNTIME_RETURN_IF_ERROR(initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, - session_options_.enable_sequential_execution)); + ONNXRUNTIME_RETURN_IF_ERROR( + initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, + node.ImplicitInputDefs(), + session_options_.enable_sequential_execution)); ONNXRUNTIME_RETURN_IF_ERROR(initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), subgraph_info.weights_buffers)); @@ -283,6 +285,12 @@ class InferenceSession::Impl { // by Compute() via OpKernelContextInternal. session_state.AddSubgraphSessionState(node.Index(), name, *subgraph_info.session_state); + // LOGS(*session_logger_, VERBOSE) << std::make_pair(subgraph_info.session_state->GetExecutionPlan(), + // &*subgraph_info.session_state); + + // recurse + ONNXRUNTIME_RETURN_IF_ERROR(InitializeSubgraphSessions(*subgraph, *subgraph_info.session_state)); + // save subgraph_info as InferenceSession owns these so they remain valid // for the entire InferenceSession. subgraph_memory_.push_back(std::move(subgraph_info)); @@ -336,7 +344,7 @@ class InferenceSession::Impl { kernel_registry_manager_, *session_logger_}; ONNXRUNTIME_RETURN_IF_ERROR(session_initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, - session_options_.enable_sequential_execution)); + {}, session_options_.enable_sequential_execution)); ONNXRUNTIME_RETURN_IF_ERROR(session_initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), weights_buffers_)); @@ -445,6 +453,7 @@ class InferenceSession::Impl { common::Status ValidateInputs(const NameMLValMap& feeds) { ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputNames(feeds)); + //TODO: It should also validate the input shapes? ONNXRUNTIME_RETURN_IF_ERROR(ValidateInputTypes(feeds)); return Status::OK(); } @@ -1111,7 +1120,7 @@ void InferenceSession::StartProfiling(const std::string& file_prefix) { } void InferenceSession::StartProfiling(const logging::Logger* custom_logger) { - impl_->StartProfiling(custom_logger); + impl_->StartProfiling(custom_logger); } std::string InferenceSession::EndProfiling() { diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index c0822c0ebe1a2..dba6daec10e52 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -13,13 +13,16 @@ #include "core/common/logging/logging.h" #include "core/common/logging/sinks/clog_sink.h" #include "core/common/status.h" +#include "core/graph/graph_base.h" #include "core/framework/allocator.h" #include "core/framework/tensor.h" #include "core/framework/ml_value.h" #include "core/framework/environment.h" #include "core/framework/tensorprotoutils.h" +#include "core/framework/onnxruntime_typeinfo.h" +#include "core/framework/onnx_object_cxx.h" #include "core/session/inference_session.h" -#include "core/graph/graph_base.h" + #include "abi_session_options_impl.h" using namespace onnxruntime::logging; @@ -41,19 +44,24 @@ using onnxruntime::common::Status; if (_status) return _status; \ } while (0) -struct ONNXEnv { - ONNXEnv(Environment* value1, LoggingManager* loggingManager1) : value(value1), loggingManager(loggingManager1) { +struct ONNXRuntimeEnv : public onnxruntime::ObjectBase { + public: + Environment* value; + LoggingManager* loggingManager; + friend class onnxruntime::ObjectBase; + + ONNXRuntimeEnv(Environment* value1, LoggingManager* loggingManager1) : value(value1), loggingManager(loggingManager1) { + ONNXRUNTIME_CHECK_C_OBJECT_LAYOUT; } /** * This function will call ::google::protobuf::ShutdownProtobufLibrary */ - ~ONNXEnv() { + ~ONNXRuntimeEnv() { + assert(ref_count == 0); delete loggingManager; delete value; } - Environment* value; - LoggingManager* loggingManager; - ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(ONNXEnv); + ONNXRUNTIME_DISALLOW_COPY_AND_ASSIGNMENT(ONNXRuntimeEnv); }; #define API_IMPL_BEGIN try { @@ -63,9 +71,9 @@ struct ONNXEnv { return CreateONNXStatus(ONNXRUNTIME_RUNTIME_EXCEPTION, ex.what()); \ } -#define TENSOR_READ_API_BEGIN \ - API_IMPL_BEGIN \ - auto v = reinterpret_cast<::onnxruntime::MLValue*>(value); \ +#define TENSOR_READ_API_BEGIN \ + API_IMPL_BEGIN \ + auto v = reinterpret_cast(value); \ auto& tensor = v->Get(); #define TENSOR_READWRITE_API_BEGIN \ @@ -92,8 +100,8 @@ class LoggingWrapper : public ISink { }; ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInitializeWithCustomLogger, ONNXRuntimeLoggingFunction logging_function, - void* logger_param, ONNXRuntimeLoggingLevel default_warning_level, _In_ const char* logid, - _Out_ ONNXEnv** out) { + _In_opt_ void* logger_param, ONNXRuntimeLoggingLevel default_warning_level, _In_ const char* logid, + _Out_ ONNXRuntimeEnv** out) { API_IMPL_BEGIN std::string name = logid; std::unique_ptr logger = std::make_unique(logging_function, logger_param); @@ -104,13 +112,13 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInitializeWithCustomLogger, ONNXRuntimeLo std::unique_ptr env; Status status = Environment::Create(env); if (status.IsOK()) - *out = new ONNXEnv(env.release(), default_logging_manager.release()); + *out = new ONNXRuntimeEnv(env.release(), default_logging_manager.release()); return ToONNXStatus(status); API_IMPL_END } ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInitialize, ONNXRuntimeLoggingLevel default_warning_level, - _In_ const char* logid, _Out_ ONNXEnv** out) { + _In_ const char* logid, _Out_ ONNXRuntimeEnv** out) { API_IMPL_BEGIN std::string name = logid; auto default_logging_manager = std::make_unique(std::unique_ptr{new CLogSink{}}, @@ -120,12 +128,12 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInitialize, ONNXRuntimeLoggingLevel defau std::unique_ptr env; Status status = Environment::Create(env); if (status.IsOK()) - *out = new ONNXEnv(env.release(), default_logging_manager.release()); + *out = new ONNXRuntimeEnv(env.release(), default_logging_manager.release()); return ToONNXStatus(status); API_IMPL_END } -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorDataLength, _In_ ONNXValuePtr value, _Out_ size_t* out) { +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorDataLength, _In_ const ONNXValue* value, _Out_ size_t* out) { TENSOR_READ_API_BEGIN const auto* src = tensor.Data(); int64_t len = tensor.Shape().Size(); @@ -141,7 +149,7 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorDataLength, _In_ ONNXValue API_IMPL_END } -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeFillStringTensor, _In_ ONNXValuePtr value, _In_ const char* s[], size_t s_len) { +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeFillStringTensor, _In_ ONNXValue* value, _In_ const char* const* s, size_t s_len) { TENSOR_READWRITE_API_BEGIN auto* dst = tensor->MutableData(); auto len = static_cast(tensor->Shape().Size()); @@ -157,8 +165,8 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeFillStringTensor, _In_ ONNXValuePtr value } template -void CreateTensorImpl(const size_t* shape, size_t shape_len, std::shared_ptr& allocator, - std::unique_ptr* out) { +ONNXStatusPtr CreateTensorImpl(const size_t* shape, size_t shape_len, ONNXRuntimeAllocatorInteface** allocator, + std::unique_ptr* out) { size_t elem_count = 1; std::vector shapes(shape_len); for (size_t i = 0; i != shape_len; ++i) { @@ -166,15 +174,25 @@ void CreateTensorImpl(const size_t* shape, size_t shape_len, std::shared_ptrAlloc(size_to_allocate); + size_t size_to_allocate; + if (!IAllocator::CalcMemSizeForArray(sizeof(T), elem_count, &size_to_allocate)) { + return CreateONNXStatus(ONNXRUNTIME_FAIL, "not enough memory"); + } + void* p_data = (*allocator)->Alloc(allocator, size_to_allocate); + if (p_data == nullptr) + return CreateONNXStatus(ONNXRUNTIME_FAIL, "size overflow"); *out = std::make_unique(DataTypeImpl::GetType(), - onnxruntime::TensorShape(shapes.data(), shape_len), + onnxruntime::TensorShape(shapes), static_cast(p_data), - allocator->Info(), - allocator); + *(*allocator)->Info(allocator), + std::make_shared(allocator)); + return nullptr; } +/** + * + * this function will create a copy of the allocator info + */ template ONNXStatusPtr CreateTensorImpl(const size_t* shape, size_t shape_len, const ONNXRuntimeAllocatorInfo* info, void* p_data, size_t p_data_len, std::unique_ptr* out) { @@ -185,23 +203,29 @@ ONNXStatusPtr CreateTensorImpl(const size_t* shape, size_t shape_len, const ONNX shapes[i] = shape[i]; } - size_t size_to_allocate = sizeof(T) * elem_count; + size_t size_to_allocate; + if (!IAllocator::CalcMemSizeForArray(sizeof(T), elem_count, &size_to_allocate)) { + return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "size overflow"); + } if (size_to_allocate > p_data_len) { std::ostringstream oss; oss << "not enough space: expected " << size_to_allocate << ", got " << p_data_len; return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, oss.str().c_str()); } *out = std::make_unique(DataTypeImpl::GetType(), - onnxruntime::TensorShape(shapes.data(), shape_len), + onnxruntime::TensorShape(shapes), p_data, *info, nullptr); return nullptr; } +/** + * this function will create a copy of the allocator info + */ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateTensorWithDataAsONNXValue, _In_ const ONNXRuntimeAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, - OnnxRuntimeTensorElementDataType type, _Out_ ONNXValuePtr* out) { + OnnxRuntimeTensorElementDataType type, _Out_ ONNXValue** out) { API_IMPL_BEGIN std::unique_ptr tensor; switch (type) { @@ -257,56 +281,55 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateTensorWithDataAsONNXValue, _In_ con value->Init(tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateTensorAsONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, _In_ const size_t* shape, size_t shape_len, OnnxRuntimeTensorElementDataType type, - _Out_ ONNXValuePtr* out) { + _Out_ ONNXValue** out) { API_IMPL_BEGIN - std::shared_ptr allocator_ = std::make_shared(allocator); std::unique_ptr tensor; switch (type) { case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: - CreateTensorImpl(shape, shape_len, allocator_, &tensor); + ONNXRUNTIME_API_RETURN_IF_ERROR(CreateTensorImpl(shape, shape_len, allocator, &tensor)); break; case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: @@ -322,31 +345,32 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateTensorAsONNXValue, _Inout_ ONNXRunt value->Init(tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } template -static ONNXStatusPtr CreateInferenceSessionImpl(_In_ ONNXEnv* env, _In_ T model_path, +static ONNXStatusPtr CreateInferenceSessionImpl(_In_ ONNXRuntimeEnv* env, _In_ T model_path, _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out) { API_IMPL_BEGIN - auto sess = std::make_unique<::onnxruntime::InferenceSession>(options->value, env->loggingManager); + auto sess = std::make_unique<::onnxruntime::InferenceSession>(options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager); Status status; - if (!options->custom_op_paths.empty()) { + if (options != nullptr && !options->custom_op_paths.empty()) { status = sess->LoadCustomOps(options->custom_op_paths); if (!status.IsOK()) return ToONNXStatus(status); } - for (ONNXRuntimeProviderFactoryPtr* p : options->provider_factories) { - ONNXRuntimeProviderPtr provider; - ONNXStatusPtr error_code = (*p)->CreateProvider(p, &provider); - if (error_code) - return error_code; - sess->RegisterExecutionProvider(std::unique_ptr( - reinterpret_cast(provider))); - } + if (options != nullptr) + for (ONNXRuntimeProviderFactoryPtr* p : options->provider_factories) { + ONNXRuntimeProviderPtr provider; + ONNXStatusPtr error_code = (*p)->CreateProvider(p, &provider); + if (error_code) + return error_code; + sess->RegisterExecutionProvider(std::unique_ptr( + reinterpret_cast(provider))); + } status = sess->Load(model_path); if (!status.IsOK()) return ToONNXStatus(status); @@ -359,14 +383,14 @@ static ONNXStatusPtr CreateInferenceSessionImpl(_In_ ONNXEnv* env, _In_ T model_ } #ifdef _WIN32 -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const wchar_t* model_path, +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXRuntimeEnv* env, _In_ const wchar_t* model_path, _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out) { API_IMPL_BEGIN return CreateInferenceSessionImpl(env, model_path, options, out); API_IMPL_END } #else -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env, _In_ const char* model_path, +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXRuntimeEnv* env, _In_ const char* model_path, _In_ const ONNXRuntimeSessionOptions* options, _Out_ ONNXSessionPtr* out) { API_IMPL_BEGIN return CreateInferenceSessionImpl(env, model_path, options, out); @@ -375,26 +399,16 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeCreateInferenceSession, _In_ ONNXEnv* env #endif ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInference, _In_ ONNXSessionPtr sess, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _In_ const char* output_names1[], size_t output_names_len, _Out_ ONNXValuePtr* output) { - API_IMPL_BEGIN - ONNXRuntimeRunOptions run_options{}; - return ONNXRuntimeRunInferenceWithRunOptions(sess, &run_options, input_names, input, input_len, - output_names1, output_names_len, output); - API_IMPL_END -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceWithRunOptions, _In_ ONNXSessionPtr sess, - _In_ ONNXRuntimeRunOptionsPtr run_options, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _In_ const char* output_names1[], size_t output_names_len, _Out_ ONNXValuePtr* output) { + _In_ ONNXRuntimeRunOptions* run_options, + _In_ const char* const* input_names, _In_ const ONNXValue* const* input, size_t input_len, + _In_ const char* const* output_names1, size_t output_names_len, _Out_ ONNXValue** output) { API_IMPL_BEGIN auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); ::onnxruntime::NameMLValMap in; const int queue_id = 0; for (size_t i = 0; i != input_len; ++i) { auto kvp = in.insert(std::make_pair(std::string(input_names[i]), - *reinterpret_cast<::onnxruntime::MLValue*>(input[i]))); + *reinterpret_cast(input[i]))); if (!kvp.second) { return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "duplicated input name"); } @@ -420,7 +434,14 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceWithRunOptions, _In_ ONNXSess fetches[i] = value; } } - auto status = session->Run(*run_options, in, output_names, &fetches); + Status status; + if (run_options == nullptr) { + ONNXRuntimeRunOptions op; + status = session->Run(op, in, output_names, &fetches); + } else { + status = session->Run(*run_options, in, output_names, &fetches); + } + if (!status.IsOK()) return ToONNXStatus(status); for (size_t i = 0; i != output_names_len; ++i) { @@ -428,60 +449,14 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceWithRunOptions, _In_ ONNXSess if (value.Fence()) value.Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); if (output[i] == nullptr) { - output[i] = reinterpret_cast(new MLValue(value)); + output[i] = reinterpret_cast(new MLValue(value)); } } return nullptr; API_IMPL_END } -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceAndFetchAll, _In_ ONNXSessionPtr sess, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len) { - API_IMPL_BEGIN - ONNXRuntimeRunOptions run_options{}; - return ONNXRuntimeRunInferenceAndFetchAllWithRunOptions(sess, &run_options, input_names, input, input_len, - output, output_len); - API_IMPL_END -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeRunInferenceAndFetchAllWithRunOptions, _In_ ONNXSessionPtr sess, - _In_ ONNXRuntimeRunOptionsPtr run_options, - _In_ const char* input_names[], _In_ ONNXValuePtr* input, size_t input_len, - _Out_ ONNXValueListPtr* output, _Out_ size_t* output_len) { - API_IMPL_BEGIN - auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); - ::onnxruntime::NameMLValMap in; - for (size_t i = 0; i != input_len; ++i) { - auto kvp = in.insert(std::make_pair(std::string(input_names[i]), - *reinterpret_cast<::onnxruntime::MLValue*>(input[i]))); - if (!kvp.second) { - return CreateONNXStatus(ONNXRUNTIME_INVALID_ARGUMENT, "duplicated input name"); - } - } - // Create output feed - std::vector output_names; - for (auto const& outp : *(session->GetModelOutputs().second)) { - output_names.push_back(outp->Name()); - } - std::vector fetches; - auto status = session->Run(*run_options, in, output_names, &fetches); - if (!status.IsOK()) - return ToONNXStatus(status); - auto* out = new MLValue[fetches.size()]; - const int queue_id = 0; - for (size_t i = 0; i != fetches.size(); ++i) { - if (fetches[i].Fence()) - fetches[i].Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); - out[i] = fetches[i]; - } - *output_len = fetches.size(); - *output = reinterpret_cast(out); - return nullptr; - API_IMPL_END -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorMutableData, _In_ ONNXValuePtr value, _Out_ void** output) { +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorMutableData, _In_ ONNXValue* value, _Out_ void** output) { TENSOR_READWRITE_API_BEGIN //TODO: test if it's a string tensor *output = tensor->MutableDataRaw(); @@ -489,66 +464,7 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorMutableData, _In_ ONNXValuePtr v API_IMPL_END } -inline OnnxRuntimeTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( - const onnxruntime::DataTypeImpl* cpp_type) { - OnnxRuntimeTensorElementDataType type; - if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; - } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; - } else { - type = ONNX_TENSOR_ELEMENT_DATA_TYPE_MAX; - } - return type; -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetTensorShapeAndType, _In_ const ONNXValuePtr value, - _Out_ ONNXRuntimeTensorTypeAndShapeInfo** out) { - TENSOR_READ_API_BEGIN - OnnxRuntimeTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor.DataType()); - if (ONNX_TENSOR_ELEMENT_DATA_TYPE_MAX == type) { - return CreateONNXStatus(ONNXRUNTIME_FAIL, "Not implemented"); - } - const onnxruntime::TensorShape& shape = tensor.Shape(); - ONNXRuntimeTensorTypeAndShapeInfo* ret = ONNXRuntimeCreateTensorTypeAndShapeInfo(); - auto status = ONNXRuntimeSetTensorElementType(ret, type); - if (status != nullptr) { - ONNXRuntimeReleaseObject(ret); - return status; - } - status = ONNXRuntimeSetDims(ret, shape.GetDims().data(), shape.GetDims().size()); - if (status != nullptr) { - ONNXRuntimeReleaseObject(ret); - return status; - } - *out = ret; - return nullptr; - API_IMPL_END -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorContent, _In_ ONNXValuePtr value, +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorContent, _In_ const ONNXValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len) { TENSOR_READ_API_BEGIN const auto* input = tensor.Data(); @@ -578,7 +494,7 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeGetStringTensorContent, _In_ ONNXValuePtr } ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeTensorProtoToONNXValue, _Inout_ ONNXRuntimeAllocator* allocator, - const void* input, int input_len, _Out_ ONNXValuePtr* out) { + const void* input, int input_len, _Out_ ONNXValue** out) { API_IMPL_BEGIN std::shared_ptr allocator_ = std::make_shared(allocator); ::ONNX_NAMESPACE::TensorProto proto; @@ -589,16 +505,11 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeTensorProtoToONNXValue, _Inout_ ONNXRunti Status st = onnxruntime::utils::TensorProtoToMLValue(proto, allocator_, nullptr, 0, *value); if (!st.IsOK()) return ToONNXStatus(st); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } -ONNXRUNTIME_API(ONNXValuePtr, ONNXRuntimeONNXValueListGetNthValue, ONNXValueListPtr list, size_t index) { - auto v = reinterpret_cast<::onnxruntime::MLValue*>(list); - return reinterpret_cast(v + index); -} - #define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(INPUT_TYPE, REAL_TYPE) \ ONNXRUNTIME_API(void, Release##INPUT_TYPE, INPUT_TYPE##Ptr value) { \ delete reinterpret_cast(value); \ @@ -609,13 +520,9 @@ ONNXRUNTIME_API(ONNXValuePtr, ONNXRuntimeONNXValueListGetNthValue, ONNXValueList delete[] reinterpret_cast(value); \ } -ONNXRUNTIME_API(void, ReleaseONNXEnv, ONNXEnv* env) { - delete env; -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out) { +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputCount, _In_ const ONNXSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN - auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + auto session = reinterpret_cast(sess); std::pair p = session->GetModelInputs(); if (!p.first.IsOK()) return ToONNXStatus(p.first); @@ -624,9 +531,9 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputCount, _In_ ONNXS API_IMPL_END } -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputCount, _In_ ONNXSessionPtr sess, _Out_ size_t* out) { +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputCount, _In_ const ONNXSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN - auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + auto session = reinterpret_cast(sess); std::pair p = session->GetModelOutputs(); if (!p.first.IsOK()) return ToONNXStatus(p.first); @@ -635,6 +542,31 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputCount, _In_ ONNX API_IMPL_END } +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct ONNXRuntimeTypeInfo** out) { + API_IMPL_BEGIN + auto session = reinterpret_cast(sess); + std::pair p = session->GetModelInputs(); + if (!p.first.IsOK()) + return ToONNXStatus(p.first); + if (p.second->size() <= index) + return CreateONNXStatus(ONNXRUNTIME_FAIL, "out of index"); + const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); + return ONNXRuntimeTypeInfo::FromDataTypeImpl(type_proto, out); + API_IMPL_END +} +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct ONNXRuntimeTypeInfo** out) { + API_IMPL_BEGIN + auto session = reinterpret_cast(sess); + std::pair p = session->GetModelOutputs(); + if (!p.first.IsOK()) + return ToONNXStatus(p.first); + if (p.second->size() <= index) + return CreateONNXStatus(ONNXRUNTIME_FAIL, "out of index"); + const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); + return ONNXRuntimeTypeInfo::FromDataTypeImpl(type_proto, out); + API_IMPL_END +} + static char* StrDup(const std::string& str, ONNXRuntimeAllocator* allocator) { char* output_string = reinterpret_cast((*allocator)->Alloc(allocator, str.size() + 1)); memcpy(output_string, str.c_str(), str.size()); @@ -642,10 +574,10 @@ static char* StrDup(const std::string& str, ONNXRuntimeAllocator* allocator) { return output_string; } -static ONNXStatusPtr GetInputOutputNameImpl(_In_ ONNXSessionPtr sess, size_t index, +static ONNXStatusPtr GetInputOutputNameImpl(_In_ const ONNXSession* sess, size_t index, _Inout_ ONNXRuntimeAllocator* allocator, bool is_input, _Out_ char** output) { - auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); + auto session = reinterpret_cast(sess); std::pair p = is_input ? session->GetModelInputs() : session->GetModelOutputs(); if (!p.first.IsOK()) return ToONNXStatus(p.first); @@ -658,8 +590,8 @@ static ONNXStatusPtr GetInputOutputNameImpl(_In_ ONNXSessionPtr sess, size_t ind return nullptr; } -ONNXRUNTIME_API(int, ONNXRuntimeIsTensor, _In_ ONNXValuePtr value) { - auto v = reinterpret_cast<::onnxruntime::MLValue*>(value); +ONNXRUNTIME_API(int, ONNXRuntimeIsTensor, _In_ const ONNXValue* value) { + auto v = reinterpret_cast(value); return v->IsTensor() ? 1 : 0; } @@ -686,14 +618,14 @@ ONNXRUNTIME_API(const struct ONNXRuntimeAllocatorInfo*, ONNXRuntimeAllocatorGetI } } -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputName, _In_ ONNXSessionPtr sess, size_t index, +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetInputName, _In_ const ONNXSession* sess, size_t index, _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** output) { API_IMPL_BEGIN return GetInputOutputNameImpl(sess, index, allocator, true, output); API_IMPL_END } -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputName, _In_ ONNXSessionPtr sess, size_t index, +ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputName, _In_ const ONNXSession* sess, size_t index, _Inout_ ONNXRuntimeAllocator* allocator, _Out_ char** output) { API_IMPL_BEGIN return GetInputOutputNameImpl(sess, index, allocator, false, output); @@ -702,5 +634,8 @@ ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeInferenceSessionGetOutputName, _In_ ONNXS DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXValue, MLValue) DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXSession, ::onnxruntime::InferenceSession) -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(ONNXValueList, ::onnxruntime::MLValue) DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(ONNXStatus, char) + +ONNXRUNTIME_API(void, ReleaseONNXEnv, ONNXRuntimeEnv* env) { + ONNXRuntimeReleaseObject(env); +} \ No newline at end of file diff --git a/onnxruntime/core/session/tensor_type_and_shape.cc b/onnxruntime/core/session/tensor_type_and_shape.cc deleted file mode 100644 index 9f707380ef35b..0000000000000 --- a/onnxruntime/core/session/tensor_type_and_shape.cc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/session/tensor_type_and_shape_c_api.h" -#include "core/framework/onnx_object.h" -#include "core/framework/tensor_shape.h" -#include -#include -#include - -struct ONNXRuntimeTensorTypeAndShapeInfo { - public: - const ONNXObject* const cls; - std::atomic_int ref_count; - OnnxRuntimeTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; - onnxruntime::TensorShape shape; - - static ONNXRuntimeTensorTypeAndShapeInfo* Create() { - return new ONNXRuntimeTensorTypeAndShapeInfo(); - } - static uint32_t ONNXRUNTIME_API_STATUSCALL ReleaseImpl(void* this_) { - ONNXRuntimeTensorTypeAndShapeInfo* this_ptr = static_cast(this_); - if (--this_ptr->ref_count == 0) - delete this_ptr; - return 0; - } - - static uint32_t ONNXRUNTIME_API_STATUSCALL AddRefImpl(void* this_) { - ONNXRuntimeTensorTypeAndShapeInfo* this_ptr = static_cast(this_); - ++this_ptr->ref_count; - return 0; - } - - private: - ONNXRuntimeTensorTypeAndShapeInfo(); - ~ONNXRuntimeTensorTypeAndShapeInfo() { - assert(ref_count == 0); - } - ONNXRuntimeTensorTypeAndShapeInfo(const ONNXRuntimeTensorTypeAndShapeInfo& other) = delete; - ONNXRuntimeTensorTypeAndShapeInfo& operator=(const ONNXRuntimeTensorTypeAndShapeInfo& other) = delete; -}; - -constexpr ONNXObject shape_cls = { - ONNXRuntimeTensorTypeAndShapeInfo::AddRefImpl, - ONNXRuntimeTensorTypeAndShapeInfo::ReleaseImpl, -}; - -#define API_IMPL_BEGIN try { -#define API_IMPL_END \ - } \ - catch (std::exception & ex) { \ - return CreateONNXStatus(ONNXRUNTIME_RUNTIME_EXCEPTION, ex.what()); \ - } - -ONNXRUNTIME_API(ONNXRuntimeTensorTypeAndShapeInfo*, ONNXRuntimeCreateTensorTypeAndShapeInfo) { - return ONNXRuntimeTensorTypeAndShapeInfo::Create(); -} - -ONNXRuntimeTensorTypeAndShapeInfo::ONNXRuntimeTensorTypeAndShapeInfo() : cls(&shape_cls), ref_count(1) { -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeSetTensorElementType, _In_ ONNXRuntimeTensorTypeAndShapeInfo* this_ptr, enum OnnxRuntimeTensorElementDataType type) { - API_IMPL_BEGIN - this_ptr->type = type; - return nullptr; - API_IMPL_END -} - -ONNXRUNTIME_API_STATUS_IMPL(ONNXRuntimeSetDims, _In_ ONNXRuntimeTensorTypeAndShapeInfo* this_ptr, _In_ const int64_t* dim_values, size_t dim_count) { - API_IMPL_BEGIN - this_ptr->shape = onnxruntime::TensorShape(dim_values, dim_count); - return nullptr; - API_IMPL_END -} - -ONNXRUNTIME_API(enum OnnxRuntimeTensorElementDataType, ONNXRuntimeGetTensorElementType, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info) { - return info->type; -} - -ONNXRUNTIME_API(size_t, ONNXRuntimeGetNumOfDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info) { - return info->shape.NumDimensions(); -} - -ONNXRUNTIME_API(void, ONNXRuntimeGetDimensions, _In_ const struct ONNXRuntimeTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) { - info->shape.CopyDims(dim_values, dim_values_length); -} - -ONNXRUNTIME_API(int64_t, ONNXRuntimeGetTensorShapeElementCount, _In_ const ONNXRuntimeTensorTypeAndShapeInfo* this_ptr) { - return this_ptr->shape.Size(); -} diff --git a/onnxruntime/test/contrib_ops/quantize_linear_test.cc b/onnxruntime/test/contrib_ops/quantize_linear_test.cc new file mode 100644 index 0000000000000..b81536aee80cd --- /dev/null +++ b/onnxruntime/test/contrib_ops/quantize_linear_test.cc @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(DequantizeLinearOpTest, DequantizeLinear_0) { + OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain); + std::vector dims{4}; + test.AddInput("x", dims, {0, 3, 128, 255}); + test.AddInput("x_scale", {}, {2.0f}); + test.AddInput("x_zero_point", {}, {128}); + test.AddOutput("y", dims, {-256.0f, -250.0f, 0.0f, 254.0f}); + test.Run(); +} + +TEST(DequantizeLinearOpTest, DequantizeLinear_1) { + OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain); + std::vector dims{4}; + test.AddInput("x", dims, {-30, -3, 100, 127}); + test.AddInput("x_scale", {}, {2.0f}); + test.AddInput("x_zero_point", {}, {-10}); + test.AddOutput("y", dims, {-40.0f, 14.0f, 220.0f, 274.0f}); + test.Run(); +} + +TEST(DequantizeLinearOpTest, DequantizeLinear_2) { + OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 1, 2, 3, + 0, 1, 2, 3, + 0, 10, 20, 30}); + test.AddAttribute("axis", 1); + test.AddInput("scale", {3}, {1.0f, 2.0f, 4.0f}); + test.AddInput("zero_point", {3}, {0, 0, 0}); + test.AddOutput("Y", dims, + {0, 1, 2, 3, + 0, 2, 4, 6, + 0, 40, 80, 120}); + test.Run(); +} + +TEST(QuantizeLinearOpTest, QuantizeLinear_0) { + OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain); + std::vector dims{6}; + test.AddInput("x", dims, {0, 2, 3, 1000, -254, -1000}); + test.AddInput("y_scale", {}, {2.0f}); + test.AddInput("y_zero_point", {}, {128}); + test.AddOutput("y", dims, {128, 129, 130, 255, 1, 0}); + test.Run(); +} + +// TODO this test is failing for Mac. needs to be debugged. +#if defined(__MACH__) +TEST(QuantizeLinearOpTest, DISABLED_QuantizeLinear_1) { +#else +TEST(QuantizeLinearOpTest, QuantizeLinear_1) { +#endif + OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 2, 3, 1000, + 0, 2, 3, 1000, + 0, 2, 3, 1000}); + test.AddAttribute("axis", 1); + test.AddInput("scale", {3}, {1, 2, 4}); + test.AddInput("zero_point", {3}, {0, 0, 0}); + test.AddOutput("Y", dims, + {0, 2, 3, 255, + 0, 1, 2, 255, + 0, 1, 1, 250}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/framework/allocation_planner_test.cc b/onnxruntime/test/framework/allocation_planner_test.cc index dbdbc5c14f0da..edfebbb931dd5 100644 --- a/onnxruntime/test/framework/allocation_planner_test.cc +++ b/onnxruntime/test/framework/allocation_planner_test.cc @@ -172,7 +172,7 @@ class PlannerTest : public ::testing::Test { onnxruntime::NodeArg* Arg(const std::string& name) { auto iter = name_to_arg_.find(name); - if (name_to_arg_.end() != iter) return iter->second; + if (name_to_arg_.end() != iter) return iter->second; return (name_to_arg_[name] = &graph_.GetOrCreateNodeArg(name, &float_type_.value)); } @@ -210,9 +210,9 @@ class PlannerTest : public ::testing::Test { } } - void CreatePlan() { + void CreatePlan(const std::vector& outer_scope_node_args = {}) { EXPECT_EQ(graph_.Resolve(), Status::OK()); - state_.SetGraphViewer(std::make_unique(graph_)); + state_.SetGraphViewer(std::make_unique(graph_)); MLValueNameIdxMap& mlvalue_name_idx_map{state_.GetMLValueNameIdxMap()}; @@ -234,7 +234,8 @@ class PlannerTest : public ::testing::Test { SequentialPlannerTestContext test_context(&shape_map_); auto status = SequentialPlanner::CreatePlan( - graph_, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, test_context, plan_); + graph_, outer_scope_node_args, execution_providers, kernel_registry_manager, + mlvalue_name_idx_map, test_context, plan_); EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); AllocationPlanTestUtility::BasicIntegrityCheck(*plan_, name_to_arg_.size()); @@ -308,30 +309,39 @@ TEST_F(PlannerTest, ChainTest) { /* InputOutputTest: Test that: (a) All inputs are classified as kPreExisting, -(b) All outputs are classified as kAllocate (in this example), -(c) Neither input nor outputs are freed. +(b) All outer scope node args are classified as kPreExisting, +(c) All outputs are classified as kAllocate (in this example), +(d) Neither input nor outputs are freed. */ TEST_F(PlannerTest, InputOutputTest) { // tensor variables: - std::string X1("X1"), X2("X2"), Y1("Y1"), Y2("Y2"); + std::string X1("X1"), X2("X2"), Y1("Y1"), Y2("Y2"), Outer1("Outer1"), Y3("Y3"); // graph structure: AddNormalNode(X1, Y1); AddNormalNode(X2, Y2); + // add node that consumes an outer scope node arg + auto outer_node = AddNormalNode(Outer1, Y3); + const NodeArg* outer_scope_node_arg = outer_node->InputDefs().at(0); + GetGraph().AddOuterScopeNodeArg(Outer1); + // simulate no shape-inference: - CreatePlan(); + CreatePlan({outer_scope_node_arg}); - // X1: kPreExisting, X2: kPreExisting, Y1: kAllocate, Y2: kAllocate + // X1: kPreExisting, X2: kPreExisting, Outer1: kPreExisting, Y1: kAllocate, Y2: kAllocate, Y3: kAllocate CheckAllocKind(X1, AllocKind::kPreExisting); CheckAllocKind(X2, AllocKind::kPreExisting); + CheckAllocKind(Outer1, AllocKind::kPreExisting); CheckAllocKind(Y1, AllocKind::kAllocateOutput); CheckAllocKind(Y2, AllocKind::kAllocateOutput); + CheckAllocKind(Y3, AllocKind::kAllocateOutput); // Nothing should be freed (since they are either inputs or outputs) CheckFreed(0, {}); CheckFreed(1, {}); + CheckFreed(2, {}); } // InPlaceTest: Check that we reuse when Inplace allows us to. diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index 3176783f08508..afeda5fdea810 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -65,7 +65,7 @@ TEST(ExecutionFrameTest, TensorAllocationTest) { std::unique_ptr p_seq_exec_plan; // TODO below line is for testing only. In production use SequentialPlanner::CreatePlan() - status = SequentialPlanner::CreatePlan(graph, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, + status = SequentialPlanner::CreatePlan(graph, {}, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, p_seq_exec_plan); EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); state.SetExecutionPlan(std::move(p_seq_exec_plan)); @@ -220,7 +220,7 @@ TEST(ExecutionFrameTest, MemPatternTest) { std::vector(6, 1.0f), &v3); std::unique_ptr p_seq_exec_plan = std::make_unique(); - status = SequentialPlanner::CreatePlan(graph, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, + status = SequentialPlanner::CreatePlan(graph, {}, execution_providers, kernel_registry_manager, mlvalue_name_idx_map, p_seq_exec_plan); EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 9d4149996bf56..04ac18692b81f 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -159,9 +159,9 @@ int real_main(int argc, char* argv[]) { usage(); return -1; } - std::unique_ptr env(nullptr, ReleaseONNXEnv); + std::unique_ptr env; { - ONNXEnv* t; + ONNXRuntimeEnv* t; ONNXStatusPtr ost = ONNXRuntimeInitialize(logging_level, "Default", &t); if (ost != nullptr) { fprintf(stderr, "Error creating environment: %s \n", ONNXRuntimeGetErrorMessage(ost)); diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc index e0684af1f222e..18968cb7d5a8a 100644 --- a/onnxruntime/test/onnx/runner.cc +++ b/onnxruntime/test/onnx/runner.cc @@ -348,7 +348,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { for (size_t i = 0; i != output_count; ++i) { output_names_raw_ptr[i] = output_names[i].c_str(); } - auto onnx_status = ONNXRuntimeRunInference(session, input_names.data(), input_values.data(), input_index, output_names_raw_ptr.data(), output_count, output_values.data()); + auto onnx_status = ONNXRuntimeRunInference(session, nullptr, input_names.data(), input_values.data(), input_index, output_names_raw_ptr.data(), output_count, output_values.data()); if (onnx_status != nullptr) { std::string onnx_runtime_error_message = ONNXRuntimeGetErrorMessage(onnx_status); ReleaseONNXStatus(onnx_status); @@ -417,7 +417,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { COMPARE_RESULT compare_result = ret.first; if (compare_result == COMPARE_RESULT::SUCCESS) { const onnx::ValueInfoProto& v = *name_output_value_info_proto[output_name]; - ret = VerifyValueInfo(v, *(MLValue*)actual_output_value); + ret = VerifyValueInfo(v, actual_output_value); compare_result = ret.first; if (compare_result != COMPARE_RESULT::SUCCESS) { switch (compare_result) { diff --git a/onnxruntime/test/onnxruntime_exec/Runtime.h b/onnxruntime/test/onnxruntime_exec/Runtime.h index c83418885b53d..9433acca0fb8f 100644 --- a/onnxruntime/test/onnxruntime_exec/Runtime.h +++ b/onnxruntime/test/onnxruntime_exec/Runtime.h @@ -273,7 +273,7 @@ class WinMLRuntime { ctensor = &output.Get(); ONNX_NAMESPACE::ValueInfoProto expected_output_info = (*outputMeta)[i]->ToProto(); - std::pair ret = VerifyValueInfo(expected_output_info, output); + std::pair ret = VerifyValueInfo(expected_output_info, (ONNXValuePtr)&output); COMPARE_RESULT compare_result = ret.first; compare_result = ret.first; if (compare_result != COMPARE_RESULT::SUCCESS) { diff --git a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc index ca9d2e5189208..965b7295d804b 100644 --- a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc @@ -34,7 +34,8 @@ class ScanOpTester : public OpTester { std::vector& graph_input_defs, std::vector& graph_output_defs, std::vector>& add_attribute_funcs) override { - // add outer_scope_0 node + // add outer_scope_0 node. push the value through an extra Identity node as a Constant gets lifted into an + // initializer which results in different treatment by the allocation planner { TypeProto float_scalar; float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); @@ -42,9 +43,9 @@ class ScanOpTester : public OpTester { mutable_dim->set_dim_value(1); { - auto& output_arg = graph.GetOrCreateNodeArg("outer_scope_0", &float_scalar); + auto& outer_scope_constant = graph.GetOrCreateNodeArg("outer_scope_constant", &float_scalar); auto* constant = graph.AddNode("outer_scope_constant", "Constant", "Constant with value kOuterNodeAddValue", - {}, {&output_arg}); + {}, {&outer_scope_constant}); TensorProto value_tensor; value_tensor.add_dims(1); @@ -52,6 +53,10 @@ class ScanOpTester : public OpTester { value_tensor.set_data_type(onnx::TensorProto_DataType_FLOAT); constant->AddAttribute("value", value_tensor); + + auto& outer_scope_node_arg = graph.GetOrCreateNodeArg("outer_scope_0", &float_scalar); + graph.AddNode("outer_scope_id", "Identity", "Identity for outer_scope_0", + {&outer_scope_constant}, {&outer_scope_node_arg}); } } @@ -179,7 +184,6 @@ static void CreateSubgraph(Graph& graph, RunOptions& options, const std::string& if (options.include_outer_scope_add) { TypeProto float_scalar; float_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); - float_scalar.mutable_tensor_type()->mutable_shape(); // create it to make checker happy but don't put any dims in it auto& outer_scope_input_arg = graph.GetOrCreateNodeArg("outer_scope_0", &float_scalar); diff --git a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c index bb5930ef3b687..3f1f105132f96 100644 --- a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c +++ b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c @@ -95,7 +95,7 @@ static int read_png_file(const char* input_file, size_t* height, size_t* width, /** * \param tensor should be a float tensor in [N,C,H,W] format */ -static int write_tensor_to_png_file(const ONNXValuePtr tensor, const char* output_file) { +static int write_tensor_to_png_file(ONNXValue* tensor, const char* output_file) { struct ONNXRuntimeTensorTypeAndShapeInfo* shape_info; ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(tensor, &shape_info)); size_t dim_count = ONNXRuntimeGetNumOfDimensions(shape_info); @@ -152,23 +152,23 @@ int run_inference(ONNXSessionPtr session, const char* input_file, const char* ou const size_t input_shape_len = sizeof(input_shape) / sizeof(input_shape[0]); const size_t model_input_len = model_input_ele_count * sizeof(float); - ONNXValuePtr input_tensor[] = {NULL}; - ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeCreateTensorWithDataAsONNXValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor[0])); - assert(input_tensor[0] != NULL); - assert(ONNXRuntimeIsTensor(input_tensor[0]) != 0); + ONNXValuePtr input_tensor = NULL; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeCreateTensorWithDataAsONNXValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); + assert(input_tensor != NULL); + assert(ONNXRuntimeIsTensor(input_tensor) != 0); ReleaseONNXRuntimeAllocatorInfo(allocator_info); const char* input_names[] = {"inputImage"}; const char* output_names[] = {"outputImage"}; - ONNXValuePtr output_tensor[] = {NULL}; - ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeRunInference(session, input_names, input_tensor, 1, output_names, 1, output_tensor)); - assert(output_tensor[0] != NULL); - assert(ONNXRuntimeIsTensor(output_tensor[0]) != 0); + ONNXValuePtr output_tensor = NULL; + ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeRunInference(session, NULL, input_names, (const ONNXValue* const*)&input_tensor, 1, output_names, 1, &output_tensor)); + assert(output_tensor != NULL); + assert(ONNXRuntimeIsTensor(output_tensor) != 0); int ret = 0; - if (write_tensor_to_png_file(output_tensor[0], output_file) != 0) { + if (write_tensor_to_png_file(output_tensor, output_file) != 0) { ret = -1; } - ReleaseONNXValue(output_tensor[0]); - ReleaseONNXValue(input_tensor[0]); + ReleaseONNXValue(output_tensor); + ReleaseONNXValue(input_tensor); free(model_input); return ret; } @@ -198,7 +198,7 @@ int main(int argc, char* argv[]) { char* model_path = argv[1]; char* input_file = argv[2]; char* output_file = argv[3]; - ONNXEnv* env; + ONNXRuntimeEnv* env; ONNXRUNTIME_ABORT_ON_ERROR(ONNXRuntimeInitialize(ONNXRUNTIME_LOGGING_LEVEL_kWARNING, "test", &env)); ONNXRuntimeSessionOptions* session_option = ONNXRuntimeCreateSessionOptions(); #ifdef USE_CUDA @@ -210,7 +210,7 @@ int main(int argc, char* argv[]) { int ret = run_inference(session, input_file, output_file); ONNXRuntimeReleaseObject(session_option); ReleaseONNXSession(session); - ReleaseONNXEnv(env); + ONNXRuntimeReleaseObject(env); if (ret != 0) { fprintf(stderr, "fail\n"); } diff --git a/onnxruntime/test/shared_lib/test_fixture.h b/onnxruntime/test/shared_lib/test_fixture.h index de3e5c90f3096..c92bf96157369 100644 --- a/onnxruntime/test/shared_lib/test_fixture.h +++ b/onnxruntime/test/shared_lib/test_fixture.h @@ -5,13 +5,21 @@ #include "core/session/onnxruntime_cxx_api.h" #include +#ifdef _WIN32 +typedef const wchar_t* PATH_TYPE; +#define TSTR(X) L##X +#else +#define TSTR(X) (X) +typedef const char* PATH_TYPE; +#endif + //empty static inline void ONNXRUNTIME_API_STATUSCALL MyLoggingFunction(void*, ONNXRuntimeLoggingLevel, const char*, const char*, const char*, const char*) { } template class CApiTestImpl : public ::testing::Test { protected: - ONNXEnv* env; + ONNXRuntimeEnv* env = nullptr; void SetUp() override { if (use_customer_logger) { @@ -22,7 +30,7 @@ class CApiTestImpl : public ::testing::Test { } void TearDown() override { - ReleaseONNXEnv(env); + if (env) ONNXRuntimeReleaseObject(env); } // Objects declared here can be used by all tests in the test case for Foo. diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 3d778ae305a1e..e7c73bb849bcf 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -13,7 +13,6 @@ using namespace onnxruntime; -template void RunSession(ONNXRuntimeAllocator* env, ONNXSession* session_object, const std::vector& dims_x, const std::vector& values_x, @@ -27,29 +26,14 @@ void RunSession(ONNXRuntimeAllocator* env, ONNXSession* session_object, ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(inputs[0], &raw_data)); memcpy(raw_data, values_x.data(), values_x.size() * sizeof(values_x[0])); std::vector input_names{"X"}; - ONNXValuePtr rtensor = nullptr; - std::unique_ptr output(nullptr, ReleaseONNXValueList); - if (with_target_names) { - const char* output_names[] = {"Y"}; - std::vector t(1); - ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeRunInference(session_object, input_names.data(), inputs.data(), inputs.size(), output_names, 1, t.data())); - rtensor = t[0]; - } else { - size_t output_len; - { - ONNXValueListPtr t; - ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeRunInferenceAndFetchAll(session_object, input_names.data(), inputs.data(), inputs.size(), &t, &output_len)); - output.reset(t); - } - - ASSERT_EQ(static_cast(1), output_len); - rtensor = ONNXRuntimeONNXValueListGetNthValue(output.get(), 0); - } - ASSERT_NE(rtensor, nullptr); + ONNXValuePtr output_tensor = nullptr; + const char* output_names[] = {"Y"}; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeRunInference(session_object, NULL, input_names.data(), inputs.data(), inputs.size(), output_names, 1, &output_tensor)); + ASSERT_NE(output_tensor, nullptr); std::unique_ptr shape_info; { ONNXRuntimeTensorTypeAndShapeInfo* shape_info_ptr; - ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(rtensor, &shape_info_ptr)); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(output_tensor, &shape_info_ptr)); shape_info.reset(shape_info_ptr); } size_t rtensor_dims = ONNXRuntimeGetNumOfDimensions(shape_info.get()); @@ -62,17 +46,15 @@ void RunSession(ONNXRuntimeAllocator* env, ONNXSession* session_object, } ASSERT_EQ(values_y.size(), total_len); float* f; - ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(rtensor, (void**)&f)); + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(output_tensor, (void**)&f)); for (size_t i = 0; i != total_len; ++i) { ASSERT_EQ(values_y[i], f[i]); } - if (with_target_names) { - ReleaseONNXValue(rtensor); - } + ReleaseONNXValue(output_tensor); } -template -void TestInference(ONNXEnv* env, const T& model_uri, +template +void TestInference(ONNXRuntimeEnv* env, T model_uri, const std::vector& dims_x, const std::vector& values_x, const std::vector& expected_dims_y, @@ -116,22 +98,14 @@ void TestInference(ONNXEnv* env, const T& model_uri, if (custom_op) { sf.AddCustomOp("libonnxruntime_custom_op_shared_lib_test.so"); } - std::unique_ptr inference_session(sf.ONNXRuntimeCreateInferenceSession(model_uri.c_str()), ReleaseONNXSession); + std::unique_ptr inference_session(sf.ONNXRuntimeCreateInferenceSession(model_uri), ReleaseONNXSession); std::unique_ptr default_allocator(MockedONNXRuntimeAllocator::Create()); // Now run - RunSession(default_allocator.get(), inference_session.get(), dims_x, values_x, expected_dims_y, expected_values_y); + RunSession(default_allocator.get(), inference_session.get(), dims_x, values_x, expected_dims_y, expected_values_y); } -#ifdef _WIN32 -typedef std::wstring PATH_TYPE; -#define TSTR(X) L##X -#else -#define TSTR(X) (X) -typedef std::string PATH_TYPE; -#endif - -static const PATH_TYPE MODEL_URI = TSTR("testdata/mul_1.pb"); -static const PATH_TYPE CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_1.pb"); +static constexpr PATH_TYPE MODEL_URI = TSTR("testdata/mul_1.pb"); +static constexpr PATH_TYPE CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_1.pb"); class CApiTestWithProvider : public CApiTest, public ::testing::WithParamInterface { @@ -139,8 +113,6 @@ class CApiTestWithProvider : public CApiTest, // Tests that the Foo::Bar() method does Abc. TEST_P(CApiTestWithProvider, simple) { - const PATH_TYPE input_filepath = TSTR("this/package/testdata/myinputfile.dat"); - const PATH_TYPE output_filepath = TSTR("this/package/testdata/myoutputfile.dat"); // simple inference test // prepare inputs std::vector dims_x = {3, 2}; @@ -150,8 +122,7 @@ TEST_P(CApiTestWithProvider, simple) { std::vector expected_dims_y = {3, 2}; std::vector expected_values_y = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; - TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), false); - TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), false); + TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), false); } INSTANTIATE_TEST_CASE_P(CApiTestWithProviders, @@ -169,11 +140,19 @@ TEST_F(CApiTest, DISABLED_custom_op) { std::vector expected_dims_y = {3, 2}; std::vector expected_values_y = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; - TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, true); - TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, true); + TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, true); } #endif +#ifdef ONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS +TEST_F(CApiTest, create_session_without_session_option) { + constexpr PATH_TYPE model_uri = TSTR("../models/opset8/test_squeezenet/model.onnx"); + ONNXSession* ret; + ONNXRUNTIME_THROW_ON_ERROR(::ONNXRuntimeCreateInferenceSession(env, model_uri, nullptr, &ret)); + ASSERT_NE(nullptr, ret); + ReleaseONNXSession(ret); +} +#endif TEST_F(CApiTest, create_tensor) { const char* s[] = {"abc", "kmp"}; size_t expected_len = 2; @@ -205,13 +184,19 @@ TEST_F(CApiTest, create_tensor_with_data) { constexpr size_t values_length = sizeof(values) / sizeof(values[0]); ONNXRuntimeAllocatorInfo* info; ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeCreateAllocatorInfo("Cpu", ONNXRuntimeDeviceAllocator, 0, ONNXRuntimeMemTypeDefault, &info)); - std::vector dims = {3}; + std::vector dims = {4}; std::unique_ptr tensor( ONNXRuntimeCreateTensorWithDataAsONNXValue(info, values, values_length * sizeof(float), dims, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT), ReleaseONNXValue); ReleaseONNXRuntimeAllocatorInfo(info); void* new_pointer; ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorMutableData(tensor.get(), &new_pointer)); ASSERT_EQ(new_pointer, values); + struct ONNXRuntimeTypeInfo* type_info; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTypeInfo(tensor.get(), &type_info)); + const struct ONNXRuntimeTensorTypeAndShapeInfo* tensor_info = ONNXRuntimeCastTypeInfoToTensorInfo(type_info); + ASSERT_NE(tensor_info, nullptr); + ASSERT_EQ(1, ONNXRuntimeGetNumOfDimensions(tensor_info)); + ONNXRuntimeReleaseObject(type_info); } int main(int argc, char** argv) { diff --git a/onnxruntime/test/shared_lib/test_io_types.cc b/onnxruntime/test/shared_lib/test_io_types.cc new file mode 100644 index 0000000000000..1b815a81f29fb --- /dev/null +++ b/onnxruntime/test/shared_lib/test_io_types.cc @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_cxx_api.h" +#include "test_fixture.h" + +using namespace onnxruntime; + +static void TestModelInfo(const ONNXSession* inference_session, bool is_input, const std::vector& dims) { + size_t input_count; + if (is_input) { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetInputCount(inference_session, &input_count)); + } else { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputCount(inference_session, &input_count)); + } + ASSERT_EQ(1, input_count); + std::unique_ptr input_type_info; + { + ONNXRuntimeTypeInfo* t; + if (is_input) { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetInputTypeInfo(inference_session, 0, &t)); + } else { + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeInferenceSessionGetOutputTypeInfo(inference_session, 0, &t)); + } + input_type_info.reset(t); + } + ASSERT_NE(nullptr, input_type_info); + const ONNXRuntimeTensorTypeAndShapeInfo* p = ONNXRuntimeCastTypeInfoToTensorInfo(input_type_info.get()); + ASSERT_NE(nullptr, p); + + enum OnnxRuntimeTensorElementDataType ele_type = ONNXRuntimeGetTensorElementType(p); + ASSERT_EQ(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, ele_type); + ASSERT_EQ(dims.size(), ONNXRuntimeGetNumOfDimensions(p)); + std::vector real_dims(dims.size()); + ONNXRuntimeGetDimensions(p, real_dims.data(), real_dims.size()); + ASSERT_EQ(real_dims, dims); +} + +TEST_F(CApiTest, input_output_type_info) { + SessionOptionsWrapper sf(env); + constexpr PATH_TYPE model_uri = TSTR("../models/opset8/test_squeezenet/model.onnx"); + std::unique_ptr inference_session(sf.ONNXRuntimeCreateInferenceSession(model_uri), ReleaseONNXSession); + TestModelInfo(inference_session.get(), true, {1, 3, 224, 224}); + TestModelInfo(inference_session.get(), false, {1, 1000, 1, 1}); +} diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc index 9ae4420c4649a..bf75a48cacf86 100644 --- a/onnxruntime/test/util/compare_mlvalue.cc +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -12,37 +12,32 @@ using namespace onnxruntime; +#define CASE_TYPE(X) \ + case ONNX_NAMESPACE::TensorProto_DataType_##X: \ + return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X; + namespace { -MLDataType ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { + +OnnxRuntimeTensorElementDataType CApiElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { switch (type) { - case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_BOOL: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_INT32: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_STRING: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_INT8: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_UINT8: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_UINT16: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_INT16: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_INT64: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_UINT32: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_UINT64: - return DataTypeImpl::GetType(); - case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: - return DataTypeImpl::GetType(); + CASE_TYPE(FLOAT) + CASE_TYPE(UINT8) + CASE_TYPE(INT8) + CASE_TYPE(UINT16) + CASE_TYPE(INT16) + CASE_TYPE(INT32) + CASE_TYPE(INT64) + CASE_TYPE(STRING) + CASE_TYPE(BOOL) + CASE_TYPE(FLOAT16) + CASE_TYPE(DOUBLE) + CASE_TYPE(UINT32) + CASE_TYPE(UINT64) + CASE_TYPE(COMPLEX64) + CASE_TYPE(COMPLEX128) + CASE_TYPE(BFLOAT16) default: - ONNXRUNTIME_NOT_IMPLEMENTED(__FUNCTION__, ":tensor type ", type, " is not supported"); + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; } } @@ -256,23 +251,35 @@ const char* ElementTypeToString(MLDataType type) { } //The expected_shape could contain unknown dimensions, but the real_shape cannot -bool AreShapesEqual(const TensorShape& real_shape, const ::ONNX_NAMESPACE::TensorShapeProto& expected_shape) { +bool AreShapesEqual(const std::vector& real_shape, const ::ONNX_NAMESPACE::TensorShapeProto& expected_shape) { const int len = expected_shape.dim_size(); - //because real_shape.NumDimensions() cannot be negative if (len < 0) return false; - if (real_shape.NumDimensions() != static_cast(len)) return false; + if (real_shape.size() != static_cast(len)) return false; for (int i = 0; i != len; ++i) { if (!expected_shape.dim(i).has_dim_value()) { - //symbolic shape, cannot validate it right now + //symbolic shape, cannot validate it right now, assume it matches every thing continue; } ::google::protobuf::int64 d = expected_shape.dim(i).dim_value(); - //dim value can be zero or negative, in such case, we assume it can match any value if (d != real_shape[i]) return false; } return true; } +template +std::ostringstream& VectorToString(const std::vector& input, std::ostringstream& oss) { + size_t len = input.size(); + oss << "["; + if (len > 0) { + oss << input[0]; + for (size_t i = 1; i != len; ++i) { + oss << ", " << input[i]; + } + } + oss << "]"; + return oss; +} + } // namespace namespace onnxruntime { @@ -306,38 +313,46 @@ std::pair CompareMLValue(const MLValue& o, const ML per_sample_tolerance, relative_per_sample_tolerance, post_processing); } -std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const MLValue& o) { - if (v.has_type()) { - if (v.type().has_tensor_type()) { - if (o.Type() != DataTypeImpl::GetType()) { - return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); - } - ::ONNX_NAMESPACE::TypeProto_Tensor t = v.type().tensor_type(); - //below code doesn't work - //if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) { - // return COMPARE_RESULT::TYPE_MISMATCH; - //} - const Tensor& o1 = o.Get(); - if (o1.DataType() != ElementTypeFromProto(t.elem_type())) { - return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); - } - if (!AreShapesEqual(o1.Shape(), t.shape())) { - std::string result; - if (!google::protobuf::TextFormat::PrintToString(t.shape(), &result)) { - result = "(unknown)"; - } - std::ostringstream oss; - oss << "Tensor shape mismatch, model file expects '" << result - << "', real output is " << o1.Shape().ToString(); - return std::make_pair(COMPARE_RESULT::SHAPE_MISMATCH, oss.str()); - } - } else { - //Cannot do this check for tensor type. - //For tensor type, o.Type() is TensorTypeBase*, but p points to a subclass of TensorTypeBase - auto p = DataTypeImpl::TypeFromProto(v.type()); - if (o.Type() != p) { - return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const ONNXValuePtr o) { + if (!v.has_type()) return std::make_pair(COMPARE_RESULT::SUCCESS, ""); + if (v.type().has_tensor_type()) { + if (ONNXRuntimeIsTensor(o) == 0) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); + } + + ::ONNX_NAMESPACE::TypeProto_Tensor t = v.type().tensor_type(); + //below code doesn't work + //if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) { + // return COMPARE_RESULT::TYPE_MISMATCH; + //} + std::unique_ptr info; + { + ONNXRuntimeTensorTypeAndShapeInfo* t1; + ONNXRUNTIME_THROW_ON_ERROR(ONNXRuntimeGetTensorShapeAndType(o, &t1)); + info.reset(t1); + } + OnnxRuntimeTensorElementDataType real_type = ONNXRuntimeGetTensorElementType(info.get()); + OnnxRuntimeTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type()); + if (real_type != expected_type) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); + } + std::vector shape = GetTensorShape(info.get()); + if (!AreShapesEqual(shape, t.shape())) { + std::string result; + if (!google::protobuf::TextFormat::PrintToString(t.shape(), &result)) { + result = "(unknown)"; } + std::ostringstream oss; + oss << "Tensor shape mismatch, model file expects '" << result << "', real output is "; + VectorToString(shape, oss); + return std::make_pair(COMPARE_RESULT::SHAPE_MISMATCH, oss.str()); + } + } else { + //Cannot do this check for tensor type. + //For tensor type, o.Type() is TensorTypeBase*, but p points to a subclass of TensorTypeBase + auto p = DataTypeImpl::TypeFromProto(v.type()); + if (((MLValue*)o)->Type() != p) { + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); } } return std::make_pair(COMPARE_RESULT::SUCCESS, ""); diff --git a/onnxruntime/test/util/include/test/compare_mlvalue.h b/onnxruntime/test/util/include/test/compare_mlvalue.h index 9f2b533f2ac47..9d2d182715e07 100644 --- a/onnxruntime/test/util/include/test/compare_mlvalue.h +++ b/onnxruntime/test/util/include/test/compare_mlvalue.h @@ -5,6 +5,7 @@ //TODO(): move compare_mlvalue.{h,cc} to test dir #include +#include #include namespace ONNX_NAMESPACE { @@ -22,5 +23,5 @@ std::pair CompareMLValue(const MLValue& real, const double relative_per_sample_tolerance, bool post_processing); //verify if the 'value' matches the 'expected' ValueInfoProto. 'value' is a model output -std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const MLValue& value); +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const ONNXValuePtr value); } // namespace onnxruntime